From be2e0c964b8925f91c26c243a47da6f8e762624b Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sun, 23 Apr 2023 16:19:16 -0400
Subject: [PATCH 01/12] Install rust-coreutils.

---
 ansible/roles/base/tasks/freebsd.yaml | 1 +
 ansible/roles/base/tasks/linux.yaml   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/ansible/roles/base/tasks/freebsd.yaml b/ansible/roles/base/tasks/freebsd.yaml
index 0be7959..5edd441 100644
--- a/ansible/roles/base/tasks/freebsd.yaml
+++ b/ansible/roles/base/tasks/freebsd.yaml
@@ -4,6 +4,7 @@
       - pstree
       - gsed
       - gmake
+      - rust-coreutils
     state: present
 
 - name: See if the alacritty termcap has been added
diff --git a/ansible/roles/base/tasks/linux.yaml b/ansible/roles/base/tasks/linux.yaml
index 943fadd..67b3554 100644
--- a/ansible/roles/base/tasks/linux.yaml
+++ b/ansible/roles/base/tasks/linux.yaml
@@ -6,6 +6,7 @@
       - linux-firmware
       - bind # dig
       - man-db
+      - uutils-coreutils
     state: present
 
 - name: Start pkgfile update service

From ba7567ad9c0a7e96fc628fc8e4f35df1ee980f5c Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 25 Apr 2023 18:33:19 -0400
Subject: [PATCH 02/12] Switch to using a script to dynamically spin up the
 netgraph bridge for jails.

---
 .../environments/laptop/host_vars/odofreebsd  |   1 -
 .../roles/firewall/files/odofreebsd_pf.conf   |  23 ++--
 .../jail/files/jail_netgraph_bridge.bash      | 120 ++++++++++++++++++
 ansible/roles/jail/files/jails/nat_dhcp.conf  |   9 +-
 ansible/roles/jail/files/jails/sample.conf    |  14 ++
 ansible/roles/jail/files/setup_netgraph_odo   |  87 -------------
 ansible/roles/jail/tasks/freebsd.yaml         |  11 ++
 7 files changed, 159 insertions(+), 106 deletions(-)
 create mode 100644 ansible/roles/jail/files/jail_netgraph_bridge.bash
 create mode 100644 ansible/roles/jail/files/jails/sample.conf
 delete mode 100644 ansible/roles/jail/files/setup_netgraph_odo

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 6a44b37..f402874 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -10,7 +10,6 @@ pflog_conf:
 network_rc: "odofreebsd_network.conf"
 rc_conf: "odofreebsd_rc.conf"
 loader_conf: "odofreebsd_loader.conf"
-netgraph_config: "setup_netgraph_odo"
 install_graphics: true
 graphics_driver: "intel"
 cputype: "intel"
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 9dc57ec..89f1832 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -1,6 +1,8 @@
-ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
-jail_net_v4 = "10.193.223.0/24"
-full_nat_v4 = "10.213.177.0/24"
+ext_if = "{ wlan0 }"
+not_ext_if = "{ !wlan0 }"
+jail_nat_v4 = "{ 10.193.223.0/24 10.213.177.0/24 10.215.1.0/24 }"
+not_jail_nat_v4 = "{ any, !10.193.223.0/24 !10.213.177.0/24 !10.215.1.0/24 }"
+dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ wgf wgh drmario colo }"
@@ -14,16 +16,13 @@ udp_pass_in = "{ 53 51820 }"
 set skip on lo
 
 # redirections
-nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
-nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0)
-
-rdr on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
-rdr on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
 # filtering
 block log all
-pass quick on $ext_if tagged ALLOWED
 pass out on $ext_if
+pass in on jail_nat
 
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
@@ -37,9 +36,3 @@ pass on $ext_if proto icmp6 all
 
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
-
-pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
-
-pass in on host_uplink0 proto udp from any to any port { 53 51820 }
-pass out on host_uplink0 proto tcp from any to any port 8081
-pass on host_uplink1
diff --git a/ansible/roles/jail/files/jail_netgraph_bridge.bash b/ansible/roles/jail/files/jail_netgraph_bridge.bash
new file mode 100644
index 0000000..a17dbd0
--- /dev/null
+++ b/ansible/roles/jail/files/jail_netgraph_bridge.bash
@@ -0,0 +1,120 @@
+#!/usr/local/bin/bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    if [ "$1" = "start" ]; then
+        shift 1
+        start_jail "${@}"
+    elif [ "$1" = "stop" ]; then
+        shift 1
+        stop_jail "${@}"
+    else
+        >&2 echo "Unrecognized command"
+        exit 1
+    fi
+}
+
+function start_jail {
+    host_interface_name="$1"
+    bridge_name="bridge_${host_interface_name}"
+    jail_interface_name="$2"
+    ip_range="$3"
+
+    assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+
+    bridge_link_name=$(detect_available_link "${bridge_name}")
+    ngctl -d -f - <<EOF
+mkpeer ${bridge_name}: eiface $bridge_link_name ether
+name ${bridge_name}:$bridge_link_name $jail_interface_name
+EOF
+    ifconfig $(ngctl msg "${jail_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${jail_interface_name}" up
+}
+
+function stop_jail {
+    host_interface_name="$1"
+    bridge_name="bridge_${host_interface_name}"
+    jail_interface_name="$2"
+
+    if ng_exists "${jail_interface_name}:"; then
+        wait_for_interface_to_exist "${jail_interface_name}" 120
+        ngctl shutdown "${jail_interface_name}:"
+    fi
+
+    if ng_exists "${bridge_name}:"; then
+        num_remaining_hooks=$(ng_bridge_get_num_hooks "${bridge_name}:")
+        if [ $num_remaining_hooks -eq 1 ]; then
+            ngctl shutdown "${bridge_name}:"
+            ngctl shutdown "${host_interface_name}:"
+        fi
+    fi
+}
+
+function assert_bridge {
+    host_interface_name="$1"
+    bridge_name="$2"
+    ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+function ng_bridge_get_num_hooks {
+    ngctl show "${1}" | grep -oE 'Num hooks: [0-9]+' | sed 's/Num hooks: //g'
+}
+
+function detect_available_link {
+    bridge_name="$1"
+    linknum=1
+    while true; do
+        link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        (>&2 echo "$link_name failed on $bridge_name")
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function wait_for_interface_to_exist {
+    # Wait for a vnet interface to exist again as a jail is shutting
+    # down. If you delete the netgraph node before the interface
+    # device exists, then the interface device will persist in a
+    # broken state.
+    ifname="$1"
+    max_wait_seconds="$2"
+    start=$(date +%s)
+    while true; do
+        now=$(date +%s)
+        if [ $((now - start)) -gt $max_wait_seconds ]; then
+            (>&2 echo "Waited for at least $max_wait_seconds seconds but the interface $ifname did not appear.")
+            return 1;
+        fi
+        if ifconfig "$ifname" >/dev/null 2>&1; then
+            return 0;
+        fi
+        sleep 2
+    done
+}
+
+main "${@}"
diff --git a/ansible/roles/jail/files/jails/nat_dhcp.conf b/ansible/roles/jail/files/jails/nat_dhcp.conf
index c4f1ba6..17609e2 100644
--- a/ansible/roles/jail/files/jails/nat_dhcp.conf
+++ b/ansible/roles/jail/files/jails/nat_dhcp.conf
@@ -1,9 +1,12 @@
 nat_dhcp {
-    path = "/jail/main/jails/nat_dhcp";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "host_link3";
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
     devfs_ruleset = 14;
-    mount.devfs; # To expose tun device
+    mount.devfs;
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";
diff --git a/ansible/roles/jail/files/jails/sample.conf b/ansible/roles/jail/files/jails/sample.conf
new file mode 100644
index 0000000..2f0ee19
--- /dev/null
+++ b/ansible/roles/jail/files/jails/sample.conf
@@ -0,0 +1,14 @@
+sample {
+    path = "/jail/main/jails/${name}";
+    vnet;
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
+    devfs_ruleset = 14;
+    mount.devfs;
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}
diff --git a/ansible/roles/jail/files/setup_netgraph_odo b/ansible/roles/jail/files/setup_netgraph_odo
deleted file mode 100644
index 1a0cef7..0000000
--- a/ansible/roles/jail/files/setup_netgraph_odo
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/usr/local/bin/bash
-
-cleanup() {
-    ngctl shutdown host_link2:
-    ngctl shutdown host_uplink0:
-    ngctl shutdown host_bridge0:
-    ngctl shutdown wg_link2:
-    ngctl shutdown wg_uplink0:
-    ngctl shutdown wg_bridge0:
-    ngctl shutdown host_link3:
-    ngctl shutdown host_uplink1:
-    ngctl shutdown host_bridge1:
-}
-
-setup_netgraph_start() {
-    cleanup
-
-    # Create a bridge for jails that only speak wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink0: bridge ether link0
-name host_uplink0:ether host_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge0: eiface link2 ether
-name host_bridge0:link2 host_link2
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
-    ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
-
-    # Create internal bridge for jails that are forced through wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook wg_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_uplink0: bridge ether link0
-name wg_uplink0:ether wg_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_bridge0: eiface link2 ether
-name wg_bridge0:link2 wg_link2
-EOF
-
-    ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
-    ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
-
-    # Create a bridge for jails given full access to NAT
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink1: bridge ether link0
-name host_uplink1:ether host_bridge1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge1: eiface link2 ether
-name host_bridge1:link2 host_link3
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink1:' getifname | grep Args | cut -d '"' -f 2) name host_uplink1 10.213.177.1/24 up
-    ifconfig $(ngctl msg 'host_bridge1:link2' getifname | grep Args | cut -d '"' -f 2) name host_link3
-
-}
-
-setup_netgraph_stop() {
-    cleanup
-}
-
-if [ "$1" = "start" ]; then
-    setup_netgraph_start
-elif [ "$1" = "stop" ]; then
-    setup_netgraph_stop
-else
-    >&2 echo "Unrecognized command"
-fi
diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index 8d1f6bc..b34afe2 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -125,6 +125,17 @@
     - src: "{{ netgraph_config }}"
       dest: /usr/local/bin/setup_netgraph
 
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: jail_netgraph_bridge.bash
+      dest: /usr/local/bin/jail_netgraph_bridge
+
 - name: Enable setup_netgraph
   when: netgraph_config is defined
   community.general.sysrc:

From 09392033774070f7e3369f7422f91ae8fd095379 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 25 Apr 2023 19:01:57 -0400
Subject: [PATCH 03/12] Switch to kea dhcp server for jail network.

---
 ansible/roles/jail_nat_dhcp/files/dhcpd.conf  | 13 ----------
 .../roles/jail_nat_dhcp/files/kea-dhcp4.conf  | 25 +++++++++++++++++++
 ansible/roles/jail_nat_dhcp/files/rc.conf     |  6 ++---
 ansible/roles/jail_nat_dhcp/files/resolv.conf |  2 +-
 .../roles/jail_nat_dhcp/tasks/freebsd.yaml    |  6 ++---
 5 files changed, 32 insertions(+), 20 deletions(-)
 delete mode 100644 ansible/roles/jail_nat_dhcp/files/dhcpd.conf
 create mode 100644 ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf

diff --git a/ansible/roles/jail_nat_dhcp/files/dhcpd.conf b/ansible/roles/jail_nat_dhcp/files/dhcpd.conf
deleted file mode 100644
index aa36935..0000000
--- a/ansible/roles/jail_nat_dhcp/files/dhcpd.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-# option definitions common to all supported networks...
-option domain-name "home.arpa";
-# option domain-name-servers ns1.home.arpa;
-option subnet-mask 255.255.255.0;
-default-lease-time 600;
-max-lease-time 7200;
-
-subnet 10.213.177.0 netmask 255.255.255.0 {
-  range 10.213.177.10 10.213.177.250;
-  option broadcast-address 10.213.177.255;
-  option routers 10.213.177.1;
-  option domain-name-servers 10.213.177.1;
-}
diff --git a/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf
new file mode 100644
index 0000000..9e09e7f
--- /dev/null
+++ b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf
@@ -0,0 +1,25 @@
+{
+    "Dhcp4": {
+        "interfaces-config": {
+            "interfaces": [ "jailnat_dhcp" ]
+        },
+        "subnet4": [
+            {
+                "subnet": "10.215.1.0/24",
+                "pools": [ { "pool": "10.215.1.10-10.215.1.250" } ],
+                "option-data": [
+                    {
+                        "name": "routers",
+                        "data": "10.215.1.1"
+                    }
+                ]
+            }
+        ],
+        "option-data": [
+            {
+                "name": "domain-name-servers",
+                "data": "10.215.1.1"
+            }
+        ]
+    }
+}
diff --git a/ansible/roles/jail_nat_dhcp/files/rc.conf b/ansible/roles/jail_nat_dhcp/files/rc.conf
index cfa78b1..e0d493a 100644
--- a/ansible/roles/jail_nat_dhcp/files/rc.conf
+++ b/ansible/roles/jail_nat_dhcp/files/rc.conf
@@ -1,3 +1,3 @@
-ifconfig_host_link3="inet 10.213.177.254 netmask 255.255.255.0"
-defaultrouter="10.213.177.1"
-dhcpd_enable="YES"
+ifconfig_jailnat_dhcp="inet 10.215.1.254 netmask 255.255.255.0"
+defaultrouter="10.215.1.1"
+kea_enable="YES"
diff --git a/ansible/roles/jail_nat_dhcp/files/resolv.conf b/ansible/roles/jail_nat_dhcp/files/resolv.conf
index 582d17e..c21685a 100644
--- a/ansible/roles/jail_nat_dhcp/files/resolv.conf
+++ b/ansible/roles/jail_nat_dhcp/files/resolv.conf
@@ -1,2 +1,2 @@
 search home.arpa
-nameserver 10.213.177.1
+nameserver 10.215.1.1
diff --git a/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml b/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml
index b98022f..0ce13f0 100644
--- a/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml
+++ b/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml
@@ -1,7 +1,7 @@
 - name: Install packages
   package:
     name:
-      - dhcpd
+      - kea
     state: present
 
 - name: Install Configuration
@@ -14,7 +14,7 @@
   loop:
     - src: rc.conf
       dest: /etc/rc.conf
-    - src: dhcpd.conf
-      dest: /usr/local/etc/dhcpd.conf
+    - src: kea-dhcp4.conf
+      dest: /usr/local/etc/kea/kea-dhcp4.conf
     - src: resolv.conf
       dest: /etc/resolv.conf

From 9fcc955d13537a930849fdf4739dbef59c89aea9 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 26 Apr 2023 01:19:01 -0400
Subject: [PATCH 04/12] Starting to write my own script to manage bhyve VMs.

vm-bhyve isn't going to allow me enough control to detect open netgraph hooks on bridges so I will manage the virtual machines myself.
---
 ansible/roles/bhyve/files/arch.conf           |  2 +-
 .../bhyve/files/bhyve_netgraph_bridge.bash    | 47 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash

diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
index 447049e..4127080 100644
--- a/ansible/roles/bhyve/files/arch.conf
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -19,7 +19,7 @@ disk0_dev="sparse-zvol"
 virt_random="yes"  # virtio-rnd
 
 # Creates a link to host_bridge1's link3 hook to the vmlink hook on a type socket
-bhyve_options="-s 2:0,virtio-net,netgraph,path=host_bridge1:,peerhook=link3"
+bhyve_options="-s 2:0,virtio-net,netgraph,path=bridge_jail_nat:,peerhook=link90"
 
 # Share a host directory to the guest via 9pfs.
 #
diff --git a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
new file mode 100644
index 0000000..a5e0ff8
--- /dev/null
+++ b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
@@ -0,0 +1,47 @@
+#!/usr/local/bin/bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    if [ "$1" = "create-disk" ]; then
+        shift 1
+        create_disk "${@}"
+    elif [ "$1" = "start" ]; then
+        shift 1
+        start_vm "${@}"
+    else
+        >&2 echo "Unrecognized command"
+        exit 1
+    fi
+}
+
+function create_disk {
+    zfs_path="$1"
+    gigabytes="$2"
+    zfs create "-V${gigabytes}G" -o volmode=dev "$zfs_path"
+}
+
+function start_vm {
+    name="$1"
+    zfs_path="$2"
+    CPU_CORES=1
+    MEMORY=1G
+    # -H release the CPU when guest issues HLT instruction. Otherwise 100% of core will be consumed.
+    bhyve \
+         -c $CPU_CORES \
+         -m $MEMORY \
+         -H \
+         -s 0,hostbridge \
+         -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
+         -s 4,virtio-blk,/dev/zvol/${zfs_path} \
+         -s 2:0,virtio-net,netgraph,path=bridge_jail_nat:,peerhook=link90 \
+         -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
+         -s 30,xhci,tablet \
+         -s 31,lpc -l com1,stdio \
+         -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \
+         "$name"
+}
+
+main "${@}"

From 3dd710eab65dcaa053cf3c9ef0d107de2dc7c9aa Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 26 Apr 2023 17:58:46 -0400
Subject: [PATCH 05/12] Start of a script for managing bhyve virtual machines.

---
 .../bhyve/files/bhyve_netgraph_bridge.bash    | 110 +++++++++++++++---
 .../roles/firewall/files/odofreebsd_pf.conf   |   3 +
 2 files changed, 98 insertions(+), 15 deletions(-)

diff --git a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
index a5e0ff8..ca90800 100644
--- a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
+++ b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
@@ -19,29 +19,109 @@ function main {
 
 function create_disk {
     zfs_path="$1"
-    gigabytes="$2"
-    zfs create "-V${gigabytes}G" -o volmode=dev "$zfs_path"
+    mount_path="$2"
+    gigabytes="$3"
+    zfs create -o "mountpoint=$mount_path" "$zfs_path"
+    cp /usr/local/share/edk2-bhyve/BHYVE_UEFI_VARS.fd "${mount_path}/"
+    tee "${mount_path}/settings" <<EOF
+CPU_CORES=1
+MEMORY=1G
+EOF
+    zfs create -s "-V${gigabytes}G" -o volmode=dev "$zfs_path/disk0"
 }
 
 function start_vm {
     name="$1"
     zfs_path="$2"
+    mount_path="$3"
+    host_interface_name="$4"
+    bridge_name="bridge_${host_interface_name}"
+    ip_range="$5"
+
+    assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+    bridge_link_name=$(detect_available_link "${bridge_name}")
+
+
     CPU_CORES=1
     MEMORY=1G
+    if [ -e "${mount_path}/settings" ]; then
+        source "${mount_path}/settings"
+    fi
     # -H release the CPU when guest issues HLT instruction. Otherwise 100% of core will be consumed.
-    bhyve \
-         -c $CPU_CORES \
-         -m $MEMORY \
-         -H \
-         -s 0,hostbridge \
-         -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
-         -s 4,virtio-blk,/dev/zvol/${zfs_path} \
-         -s 2:0,virtio-net,netgraph,path=bridge_jail_nat:,peerhook=link90 \
-         -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
-         -s 30,xhci,tablet \
-         -s 31,lpc -l com1,stdio \
-         -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \
-         "$name"
+         # -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
+             # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
+    while true; do
+        set -x
+        bhyve \
+            -c $CPU_CORES \
+            -m $MEMORY \
+            -H \
+            -s 0,hostbridge \
+            -s "4,nvme,/dev/zvol/${zfs_path}/disk0" \
+            -s "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name}" \
+            -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080 \
+            -s 30,xhci,tablet \
+            -s 31,lpc -l com1,stdio \
+            -l "bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,${mount_path}/BHYVE_UEFI_VARS.fd" \
+            "$name"
+        exit_code=$?
+        set +x
+        if [ $exit_code -eq 0 ]; then
+            echo "Rebooting."
+        elif [ $exit_code -eq 1 ]; then
+            echo "Powered off."
+            break
+        elif [ $exit_code -eq 2 ]; then
+            echo "Halted."
+            break
+        elif [ $exit_code -eq 3 ]; then
+            echo "Triple fault."
+            break
+        elif [ $exit_code -eq 4 ]; then
+            echo "Exited due to an error."
+            break
+        fi
+    done
 }
 
+function detect_available_link {
+    bridge_name="$1"
+    linknum=1
+    while true; do
+        link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function assert_bridge {
+    host_interface_name="$1"
+    bridge_name="$2"
+    ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+
 main "${@}"
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 89f1832..95897f9 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -22,7 +22,10 @@ rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1
 # filtering
 block log all
 pass out on $ext_if
+
 pass in on jail_nat
+# Allow traffic from my machine to the jails/virtual machines
+pass out on jail_nat from jail_nat
 
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a

From e43fa0eab38e414f8a9184792e23610f16ece662 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 26 Apr 2023 18:17:35 -0400
Subject: [PATCH 06/12] Destroy the bhyve vm after exit.

---
 ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
index ca90800..7741002 100644
--- a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
+++ b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
@@ -52,7 +52,9 @@ function start_vm {
              # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
     while true; do
         set -x
+        set +e
         bhyve \
+            -D \
             -c $CPU_CORES \
             -m $MEMORY \
             -H \
@@ -65,6 +67,7 @@ function start_vm {
             -l "bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,${mount_path}/BHYVE_UEFI_VARS.fd" \
             "$name"
         exit_code=$?
+        set -e
         set +x
         if [ $exit_code -eq 0 ]; then
             echo "Rebooting."
@@ -82,6 +85,9 @@ function start_vm {
             break
         fi
     done
+
+    bhyvectl "--vm=$name" --destroy
+    echo "Destroyed bhyve vm."
 }
 
 function detect_available_link {

From 310837ddb3bc940c36ecc70c1471a1c3ce25a281 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 26 Apr 2023 19:06:28 -0400
Subject: [PATCH 07/12] Use statically-generated mac addresses.

---
 ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash | 11 ++++++++++-
 ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf     |  8 +++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
index 7741002..23db06c 100644
--- a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
+++ b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
@@ -38,6 +38,8 @@ function start_vm {
     bridge_name="bridge_${host_interface_name}"
     ip_range="$5"
 
+    mac_address=$(calculate_mac_address "$name")
+
     assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
     bridge_link_name=$(detect_available_link "${bridge_name}")
 
@@ -60,7 +62,8 @@ function start_vm {
             -H \
             -s 0,hostbridge \
             -s "4,nvme,/dev/zvol/${zfs_path}/disk0" \
-            -s "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name}" \
+            -s "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}" \
+            -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
             -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080 \
             -s 30,xhci,tablet \
             -s 31,lpc -l com1,stdio \
@@ -129,5 +132,11 @@ function ng_exists {
     ngctl status "${1}" >/dev/null 2>&1
 }
 
+function calculate_mac_address {
+    name="$1"
+    source=$(md5 -r -s "$name" | awk '{print $1}')
+    echo "${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}:${source:10:2}"
+}
+
 
 main "${@}"
diff --git a/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf
index 9e09e7f..ce2043c 100644
--- a/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf
+++ b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf
@@ -6,12 +6,18 @@
         "subnet4": [
             {
                 "subnet": "10.215.1.0/24",
-                "pools": [ { "pool": "10.215.1.10-10.215.1.250" } ],
+                "pools": [ { "pool": "10.215.1.10-10.215.1.200" } ],
                 "option-data": [
                     {
                         "name": "routers",
                         "data": "10.215.1.1"
                     }
+                ],
+                "reservations": [
+                    {
+                        "hw-address": "d6:19:4c:68:fc:c7",
+                        "ip-address": "10.215.1.201"
+                    }
                 ]
             }
         ],

From 15d5c73b5bbb960016f105e55cd56be157fc6acd Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 26 Apr 2023 19:37:02 -0400
Subject: [PATCH 08/12] Add commented out jaeger config because it is not
 working.

---
 ansible/roles/firewall/files/odofreebsd_pf.conf | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 95897f9..516cf16 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -19,6 +19,10 @@ set skip on lo
 nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
 rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
+# Redirect jaeger ports to virtual machine.
+# nat pass on lo inet from 127.0.0.0/24 to 127.0.0.0/24 port {6831 6832 16686 14268} -> (jail_nat)
+# rdr pass proto {tcp, udp} from jail_nat:network to 127.0.0.0/24 port {6831 6832 16686 14268} -> 10.215.1.201
+
 # filtering
 block log all
 pass out on $ext_if
@@ -31,7 +35,7 @@ pass out on jail_nat from jail_nat
 # them because changes to pass rules will update when running a
 # `service pf reload` but interfaces that we `skip` will not update (I
 # forget if its from adding, removing, or both. TODO: test to figure
-# it out)
+# it out). Also skipped interfaces are not subject to nat/rdr rules.
 pass quick on $allow
 
 pass on $ext_if proto icmp all

From 5da4f8fb99f7bb5ffcaf750301f9e01dca633970 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Thu, 27 Apr 2023 10:55:26 -0400
Subject: [PATCH 09/12] Install the bhyve management script.

---
 ansible/roles/bhyve/tasks/freebsd.yaml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml
index dc5b243..7da5e07 100644
--- a/ansible/roles/bhyve/tasks/freebsd.yaml
+++ b/ansible/roles/bhyve/tasks/freebsd.yaml
@@ -13,6 +13,17 @@
       - bhyve-firmware # For UEFI
     state: present
 
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: bhyve_netgraph_bridge.bash
+      dest: /usr/local/bin/bhyve_netgraph_bridge
+
 - name: Create zfs dataset
   zfs:
     name: "{{ bhyve_dataset }}"

From 24bfa840ffd7b75a31255e332fa86f6ffde8d54a Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Thu, 27 Apr 2023 14:56:32 -0400
Subject: [PATCH 10/12] Starting to transition the home server to the new
 dynamic netgraph devices.

---
 .../roles/firewall/files/homeserver_pf.conf   | 24 ++---
 .../roles/firewall/files/odofreebsd_pf.conf   |  6 +-
 .../jail/files/setup_netgraph_homeserver      | 95 ++++++++-----------
 .../roles/package_manager/tasks/freebsd.yaml  | 12 +--
 4 files changed, 58 insertions(+), 79 deletions(-)

diff --git a/ansible/roles/firewall/files/homeserver_pf.conf b/ansible/roles/firewall/files/homeserver_pf.conf
index 9653cbf..dee2303 100644
--- a/ansible/roles/firewall/files/homeserver_pf.conf
+++ b/ansible/roles/firewall/files/homeserver_pf.conf
@@ -1,6 +1,7 @@
 ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
-jail_net_v4 = "10.193.223.0/24"
-full_nat_v4 = "10.213.177.0/24"
+not_ext_if = "{ !igb0 !igb1 !ix0 !ix1 !wlan0 }"
+jail_nat_v4 = "{ 10.215.1.0/24 }"
+not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ wgh wgf }"
@@ -15,11 +16,8 @@ unifi_ports = "{ 8443 3478 10001 8080 1900 8843 8880 6789 5514 }"
 set skip on lo
 
 # redirections
-nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
-nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0)
-
-rdr pass on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
-rdr pass on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
 nat pass on host_uplink0 inet proto tcp from any to any port 8081 tag ALLOWED -> (host_uplink0)
 rdr pass on $ext_if inet proto tcp from any to any port 8081 tag ALLOWED -> 10.193.223.20 port 8081
@@ -32,6 +30,10 @@ rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports tag
 block log all
 pass out on $ext_if
 
+pass in on jail_nat
+# Allow traffic from my machine to the jails/virtual machines
+pass out on jail_nat from $jail_nat_v4
+
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
 # `service pf reload` but interfaces that we `skip` will not update (I
@@ -45,8 +47,6 @@ pass on $ext_if proto icmp6 all
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
 
-pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
-
-pass in on host_uplink0 proto udp from any to any port { 53 51820 }
-pass out on host_uplink0 proto tcp from any to any port 8081
-pass on host_uplink1
+# pass in on host_uplink0 proto udp from any to any port { 53 51820 }
+# pass out on host_uplink0 proto tcp from any to any port 8081
+# pass on host_uplink1
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 516cf16..559cf9a 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -1,7 +1,7 @@
 ext_if = "{ wlan0 }"
 not_ext_if = "{ !wlan0 }"
-jail_nat_v4 = "{ 10.193.223.0/24 10.213.177.0/24 10.215.1.0/24 }"
-not_jail_nat_v4 = "{ any, !10.193.223.0/24 !10.213.177.0/24 !10.215.1.0/24 }"
+jail_nat_v4 = "{ 10.215.1.0/24 }"
+not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
 dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }"
 
 dhcp = "{ bootpc, bootps }"
@@ -29,7 +29,7 @@ pass out on $ext_if
 
 pass in on jail_nat
 # Allow traffic from my machine to the jails/virtual machines
-pass out on jail_nat from jail_nat
+pass out on jail_nat from $jail_nat_v4
 
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
diff --git a/ansible/roles/jail/files/setup_netgraph_homeserver b/ansible/roles/jail/files/setup_netgraph_homeserver
index 1a0cef7..2b2d583 100644
--- a/ansible/roles/jail/files/setup_netgraph_homeserver
+++ b/ansible/roles/jail/files/setup_netgraph_homeserver
@@ -1,76 +1,55 @@
 #!/usr/local/bin/bash
 
 cleanup() {
-    ngctl shutdown host_link2:
-    ngctl shutdown host_uplink0:
-    ngctl shutdown host_bridge0:
-    ngctl shutdown wg_link2:
-    ngctl shutdown wg_uplink0:
-    ngctl shutdown wg_bridge0:
-    ngctl shutdown host_link3:
-    ngctl shutdown host_uplink1:
-    ngctl shutdown host_bridge1:
+    # ngctl shutdown host_link2:
+    # ngctl shutdown host_uplink0:
+    # ngctl shutdown host_bridge0:
+    # ngctl shutdown wg_link2:
+    # ngctl shutdown wg_uplink0:
+    # ngctl shutdown wg_bridge0:
+    echo ""
 }
 
 setup_netgraph_start() {
     cleanup
 
-    # Create a bridge for jails that only speak wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink0
-EOF
+#     # Create a bridge for jails that only speak wireguard
+#     ngctl -d -f - <<EOF
+# mkpeer . eiface hook ether
+# name .:hook host_uplink0
+# EOF
 
-    ngctl -d -f - <<EOF
-mkpeer host_uplink0: bridge ether link0
-name host_uplink0:ether host_bridge0
-EOF
+#     ngctl -d -f - <<EOF
+# mkpeer host_uplink0: bridge ether link0
+# name host_uplink0:ether host_bridge0
+# EOF
 
-    ngctl -d -f - <<EOF
-mkpeer host_bridge0: eiface link2 ether
-name host_bridge0:link2 host_link2
-EOF
+#     ngctl -d -f - <<EOF
+# mkpeer host_bridge0: eiface link2 ether
+# name host_bridge0:link2 host_link2
+# EOF
 
-    ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
-    ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
+#     ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
+#     ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
 
-    # Create internal bridge for jails that are forced through wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook wg_uplink0
-EOF
+#     # Create internal bridge for jails that are forced through wireguard
+#     ngctl -d -f - <<EOF
+# mkpeer . eiface hook ether
+# name .:hook wg_uplink0
+# EOF
 
-    ngctl -d -f - <<EOF
-mkpeer wg_uplink0: bridge ether link0
-name wg_uplink0:ether wg_bridge0
-EOF
+#     ngctl -d -f - <<EOF
+# mkpeer wg_uplink0: bridge ether link0
+# name wg_uplink0:ether wg_bridge0
+# EOF
 
-    ngctl -d -f - <<EOF
-mkpeer wg_bridge0: eiface link2 ether
-name wg_bridge0:link2 wg_link2
-EOF
+#     ngctl -d -f - <<EOF
+# mkpeer wg_bridge0: eiface link2 ether
+# name wg_bridge0:link2 wg_link2
+# EOF
 
-    ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
-    ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
-
-    # Create a bridge for jails given full access to NAT
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink1: bridge ether link0
-name host_uplink1:ether host_bridge1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge1: eiface link2 ether
-name host_bridge1:link2 host_link3
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink1:' getifname | grep Args | cut -d '"' -f 2) name host_uplink1 10.213.177.1/24 up
-    ifconfig $(ngctl msg 'host_bridge1:link2' getifname | grep Args | cut -d '"' -f 2) name host_link3
+#     ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
+#     ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
 
 }
 
diff --git a/ansible/roles/package_manager/tasks/freebsd.yaml b/ansible/roles/package_manager/tasks/freebsd.yaml
index c98f0c0..ce9b8b3 100644
--- a/ansible/roles/package_manager/tasks/freebsd.yaml
+++ b/ansible/roles/package_manager/tasks/freebsd.yaml
@@ -8,6 +8,12 @@
   loop:
     - /usr/local/etc/pkg
     - /usr/local/etc/pkg/repos
+    
+- name: Install packages
+  package:
+    name:
+      - pkg-provides
+    state: present
 
 - name: Install Configuration
   register: changed_config
@@ -26,9 +32,3 @@
 # - name: Replace all packages with packages from new repo
 #   command: pkg upgrade -f -y
 #   when: changed_config.changed
-
-- name: Install packages
-  package:
-    name:
-      - pkg-provides
-    state: present

From 886776193987d518132c5268ad166f03138006bd Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Thu, 27 Apr 2023 16:58:06 -0400
Subject: [PATCH 11/12] Transition the home server to the dynamic netgraph
 devices.

---
 .../roles/firewall/files/homeserver_pf.conf   | 18 +++----
 ansible/roles/jail/files/jails/cloak.conf     |  9 ++--
 ansible/roles/jail/files/jails/dagger.conf    |  4 +-
 .../jail/files/setup_netgraph_homeserver      | 47 +------------------
 4 files changed, 20 insertions(+), 58 deletions(-)

diff --git a/ansible/roles/firewall/files/homeserver_pf.conf b/ansible/roles/firewall/files/homeserver_pf.conf
index dee2303..933fea2 100644
--- a/ansible/roles/firewall/files/homeserver_pf.conf
+++ b/ansible/roles/firewall/files/homeserver_pf.conf
@@ -19,12 +19,16 @@ set skip on lo
 nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
 rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
-nat pass on host_uplink0 inet proto tcp from any to any port 8081 tag ALLOWED -> (host_uplink0)
-rdr pass on $ext_if inet proto tcp from any to any port 8081 tag ALLOWED -> 10.193.223.20 port 8081
+# cloak
+nat pass on $ext_if inet from 10.215.2.0/24 to !10.215.2.0/24 -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.2.1 port 53 -> 1.1.1.1 port 53
+
+nat pass on restricted_nat inet proto tcp from any to any port 8081 -> (restricted_nat)
+rdr pass on $ext_if inet proto tcp from any to any port 8081 -> 10.215.2.2 port 8081
 
 # Forward ports for unifi controller
-rdr pass on $ext_if inet proto tcp from any to any port 65022 tag ALLOWED -> 10.213.177.8 port 22
-rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports tag ALLOWED -> 10.213.177.8
+rdr pass on $ext_if inet proto tcp from any to any port 65022 -> 10.213.177.8 port 22
+rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports -> 10.213.177.8
 
 # filtering
 block log all
@@ -34,6 +38,8 @@ pass in on jail_nat
 # Allow traffic from my machine to the jails/virtual machines
 pass out on jail_nat from $jail_nat_v4
 
+pass in on restricted_nat proto {udp, tcp} from any to any port { 53 51820 }
+
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
 # `service pf reload` but interfaces that we `skip` will not update (I
@@ -46,7 +52,3 @@ pass on $ext_if proto icmp6 all
 
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
-
-# pass in on host_uplink0 proto udp from any to any port { 53 51820 }
-# pass out on host_uplink0 proto tcp from any to any port 8081
-# pass on host_uplink1
diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
index 181974f..dac8985 100644
--- a/ansible/roles/jail/files/jails/cloak.conf
+++ b/ansible/roles/jail/files/jails/cloak.conf
@@ -1,8 +1,11 @@
 cloak {
-    path = "/jail/main/jails/cloak";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "host_link2";
-    vnet.interface += "wg_uplink0";
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start restricted_nat jail${name} 10.215.2.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop restricted_nat jail${name}";
+    vnet.interface += "jail${name}";
+    vnet.interface += "cloak";
+
     devfs_ruleset = 13;
     mount.devfs; # To expose tun device
 
diff --git a/ansible/roles/jail/files/jails/dagger.conf b/ansible/roles/jail/files/jails/dagger.conf
index cab230e..97034c8 100644
--- a/ansible/roles/jail/files/jails/dagger.conf
+++ b/ansible/roles/jail/files/jails/dagger.conf
@@ -1,7 +1,7 @@
 dagger {
-    path = "/jail/main/jails/dagger";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "wg_link2";
+    vnet.interface += "dagger";
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";
diff --git a/ansible/roles/jail/files/setup_netgraph_homeserver b/ansible/roles/jail/files/setup_netgraph_homeserver
index 2b2d583..cfa53bf 100644
--- a/ansible/roles/jail/files/setup_netgraph_homeserver
+++ b/ansible/roles/jail/files/setup_netgraph_homeserver
@@ -1,56 +1,13 @@
 #!/usr/local/bin/bash
 
 cleanup() {
-    # ngctl shutdown host_link2:
-    # ngctl shutdown host_uplink0:
-    # ngctl shutdown host_bridge0:
-    # ngctl shutdown wg_link2:
-    # ngctl shutdown wg_uplink0:
-    # ngctl shutdown wg_bridge0:
-    echo ""
+    /usr/local/bin/jail_netgraph_bridge stop cloak dagger
 }
 
 setup_netgraph_start() {
     cleanup
 
-#     # Create a bridge for jails that only speak wireguard
-#     ngctl -d -f - <<EOF
-# mkpeer . eiface hook ether
-# name .:hook host_uplink0
-# EOF
-
-#     ngctl -d -f - <<EOF
-# mkpeer host_uplink0: bridge ether link0
-# name host_uplink0:ether host_bridge0
-# EOF
-
-#     ngctl -d -f - <<EOF
-# mkpeer host_bridge0: eiface link2 ether
-# name host_bridge0:link2 host_link2
-# EOF
-
-#     ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
-#     ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
-
-#     # Create internal bridge for jails that are forced through wireguard
-#     ngctl -d -f - <<EOF
-# mkpeer . eiface hook ether
-# name .:hook wg_uplink0
-# EOF
-
-#     ngctl -d -f - <<EOF
-# mkpeer wg_uplink0: bridge ether link0
-# name wg_uplink0:ether wg_bridge0
-# EOF
-
-#     ngctl -d -f - <<EOF
-# mkpeer wg_bridge0: eiface link2 ether
-# name wg_bridge0:link2 wg_link2
-# EOF
-
-#     ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
-#     ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
-
+    /usr/local/bin/jail_netgraph_bridge start cloak dagger 192.168.1.0/24
 }
 
 setup_netgraph_stop() {

From 04a8ba064fcf15904b610b22fd1391a71f0bb8f9 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Thu, 27 Apr 2023 17:02:21 -0400
Subject: [PATCH 12/12] This nat rule doesn't seem necessary.

---
 ansible/roles/firewall/files/homeserver_pf.conf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ansible/roles/firewall/files/homeserver_pf.conf b/ansible/roles/firewall/files/homeserver_pf.conf
index 933fea2..75dd668 100644
--- a/ansible/roles/firewall/files/homeserver_pf.conf
+++ b/ansible/roles/firewall/files/homeserver_pf.conf
@@ -23,7 +23,6 @@ rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1
 nat pass on $ext_if inet from 10.215.2.0/24 to !10.215.2.0/24 -> (wlan0)
 rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.2.1 port 53 -> 1.1.1.1 port 53
 
-nat pass on restricted_nat inet proto tcp from any to any port 8081 -> (restricted_nat)
 rdr pass on $ext_if inet proto tcp from any to any port 8081 -> 10.215.2.2 port 8081
 
 # Forward ports for unifi controller