From fbfadadffcddf02ec73e9b07068505eafad1df5a Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 18:53:03 -0400
Subject: [PATCH 01/10] Start a bhyve role.

---
 .../environments/laptop/host_vars/odofreebsd  |  1 +
 ansible/playbook.yaml                         | 51 ++++++++++---------
 ansible/roles/bhyve/defaults/main.yaml        |  1 +
 ansible/roles/bhyve/tasks/common.yaml         | 14 +++++
 ansible/roles/bhyve/tasks/freebsd.yaml        | 22 ++++++++
 ansible/roles/bhyve/tasks/linux.yaml          |  6 +++
 ansible/roles/bhyve/tasks/main.yaml           |  2 +
 ansible/roles/bhyve/tasks/peruser.yaml        | 29 +++++++++++
 .../roles/bhyve/tasks/peruser_freebsd.yaml    |  0
 ansible/roles/bhyve/tasks/peruser_linux.yaml  |  0
 10 files changed, 101 insertions(+), 25 deletions(-)
 create mode 100644 ansible/roles/bhyve/defaults/main.yaml
 create mode 100644 ansible/roles/bhyve/tasks/common.yaml
 create mode 100644 ansible/roles/bhyve/tasks/freebsd.yaml
 create mode 100644 ansible/roles/bhyve/tasks/linux.yaml
 create mode 100644 ansible/roles/bhyve/tasks/main.yaml
 create mode 100644 ansible/roles/bhyve/tasks/peruser.yaml
 create mode 100644 ansible/roles/bhyve/tasks/peruser_freebsd.yaml
 create mode 100644 ansible/roles/bhyve/tasks/peruser_linux.yaml

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index ba98b33..9c383e3 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -44,3 +44,4 @@ jail_list:
   - name: dagger
     conf:
       src: dagger
+bhyve_dataset: zroot/freebsd/release/vm
diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index ce84353..9e387b6 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -2,28 +2,29 @@
   vars:
     ansible_become: True
   roles:
-    - sudo
-    - users
-    - package_manager
-    - zrepl
-    - zsh
-    - network
-    - sshd
-    - base
-    - firewall
-    - cpu
-    - ntp
-    - build
-    - graphics
-    - gpg
-    - fonts
-    - alacritty
-    - sway
-    - emacs
-    - firefox
-    - devfs
-    - ssh_client
-    - jail
-    - fuse
-    - autofs
-    - exfat
+    # - sudo
+    # - users
+    # - package_manager
+    # - zrepl
+    # - zsh
+    # - network
+    # - sshd
+    # - base
+    # - firewall
+    # - cpu
+    # - ntp
+    # - build
+    # - graphics
+    # - gpg
+    # - fonts
+    # - alacritty
+    # - sway
+    # - emacs
+    # - firefox
+    # - devfs
+    # - ssh_client
+    # - jail
+    # - fuse
+    # - autofs
+    # - exfat
+    - bhyve
diff --git a/ansible/roles/bhyve/defaults/main.yaml b/ansible/roles/bhyve/defaults/main.yaml
new file mode 100644
index 0000000..1838e9c
--- /dev/null
+++ b/ansible/roles/bhyve/defaults/main.yaml
@@ -0,0 +1 @@
+bhyve_mountpoint: "/vm"
diff --git a/ansible/roles/bhyve/tasks/common.yaml b/ansible/roles/bhyve/tasks/common.yaml
new file mode 100644
index 0000000..d7c1735
--- /dev/null
+++ b/ansible/roles/bhyve/tasks/common.yaml
@@ -0,0 +1,14 @@
+- import_tasks: tasks/freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/linux.yaml
+  when: 'os_flavor == "linux"'
+
+- include_tasks:
+    file: tasks/peruser.yaml
+    apply:
+      become: yes
+      become_user: "{{ initialize_user }}"
+  loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}"
+  loop_control:
+    loop_var: initialize_user
diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml
new file mode 100644
index 0000000..bbdf85f
--- /dev/null
+++ b/ansible/roles/bhyve/tasks/freebsd.yaml
@@ -0,0 +1,22 @@
+#
+# Create a new VM:
+#   vm iso 'http://mirror.clarkson.edu/archlinux/iso/2022.01.01/archlinux-2022.01.01-x86_64.iso'
+#   vm create -t arch -s 50G testvm
+#   vm install testvm 'archlinux-2022.01.01-x86_64.iso'
+#
+#
+- name: Install packages
+  package:
+    name:
+      - vm-bhyve
+      - tmux # for interactive consoles
+      - bhyve-firmware # For UEFI
+      # - uefi-edk2-bhyve  # Other UEFI firmware?
+    state: present
+
+- name: Create zfs dataset
+  zfs:
+    name: "{{ bhyve_dataset }}"
+    state: present
+    extra_zfs_properties:
+      mountpoint: "{{ bhyve_mountpoint }}"
diff --git a/ansible/roles/bhyve/tasks/linux.yaml b/ansible/roles/bhyve/tasks/linux.yaml
new file mode 100644
index 0000000..e1835f0
--- /dev/null
+++ b/ansible/roles/bhyve/tasks/linux.yaml
@@ -0,0 +1,6 @@
+# - name: Install packages
+#   pacman:
+#     name:
+#       - foo
+#     state: present
+#     update_cache: true
diff --git a/ansible/roles/bhyve/tasks/main.yaml b/ansible/roles/bhyve/tasks/main.yaml
new file mode 100644
index 0000000..639ba76
--- /dev/null
+++ b/ansible/roles/bhyve/tasks/main.yaml
@@ -0,0 +1,2 @@
+- import_tasks: tasks/common.yaml
+  when: os_flavor == "freebsd" and bhyve_dataset is defined
diff --git a/ansible/roles/bhyve/tasks/peruser.yaml b/ansible/roles/bhyve/tasks/peruser.yaml
new file mode 100644
index 0000000..111e886
--- /dev/null
+++ b/ansible/roles/bhyve/tasks/peruser.yaml
@@ -0,0 +1,29 @@
+- include_role:
+    name: per_user
+
+# - name: Create directories
+#   file:
+#     name: "{{ account_homedir.stdout }}/{{ item }}"
+#     state: directory
+#     mode: 0700
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - ".config/foo"
+
+# - name: Copy files
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ account_homedir.stdout }}/{{ item.dest }}"
+#     mode: 0600
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - src: foo.conf
+#       dest: .config/foo/foo.conf
+
+- import_tasks: tasks/peruser_freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/peruser_linux.yaml
+  when: 'os_flavor == "linux"'
diff --git a/ansible/roles/bhyve/tasks/peruser_freebsd.yaml b/ansible/roles/bhyve/tasks/peruser_freebsd.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/ansible/roles/bhyve/tasks/peruser_linux.yaml b/ansible/roles/bhyve/tasks/peruser_linux.yaml
new file mode 100644
index 0000000..e69de29

From a344c92d8374d771b1d5aa8ab040e5c6c49ff783 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 19:06:44 -0400
Subject: [PATCH 02/10] Configure bhyve.

---
 .../environments/laptop/host_vars/odofreebsd  |  1 +
 ansible/roles/bhyve/defaults/main.yaml        |  1 +
 ansible/roles/bhyve/files/arch.conf           | 17 ++++++++++
 ansible/roles/bhyve/tasks/freebsd.yaml        | 31 +++++++++++++++++++
 4 files changed, 50 insertions(+)
 create mode 100644 ansible/roles/bhyve/files/arch.conf

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 9c383e3..bb1a3c8 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -45,3 +45,4 @@ jail_list:
     conf:
       src: dagger
 bhyve_dataset: zroot/freebsd/release/vm
+bhyve_list: []
diff --git a/ansible/roles/bhyve/defaults/main.yaml b/ansible/roles/bhyve/defaults/main.yaml
index 1838e9c..d7cab58 100644
--- a/ansible/roles/bhyve/defaults/main.yaml
+++ b/ansible/roles/bhyve/defaults/main.yaml
@@ -1 +1,2 @@
 bhyve_mountpoint: "/vm"
+bhyve_list: []
diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
new file mode 100644
index 0000000..1eec7e2
--- /dev/null
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -0,0 +1,17 @@
+# Use UEFI when booting from a disk
+loader="uefi"
+graphics="yes"
+xhci_mouse="yes"
+graphics_listen="127.0.0.1"
+#graphics_listen="10.216.1.1"
+graphics_res="1920x1080"
+#prestart=""
+
+cpu=1
+memory=1024M
+network0_type="virtio-net"
+network0_switch="customswitch"
+disk0_type="virtio-blk"
+disk0_name="disk0"
+disk0_dev="sparse-zvol"
+virt_random="yes"  # virtio-rnd
diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml
index bbdf85f..8f73216 100644
--- a/ansible/roles/bhyve/tasks/freebsd.yaml
+++ b/ansible/roles/bhyve/tasks/freebsd.yaml
@@ -20,3 +20,34 @@
     state: present
     extra_zfs_properties:
       mountpoint: "{{ bhyve_mountpoint }}"
+
+- name: Enable bhyve
+  community.general.sysrc:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    path: /etc/rc.conf.d/vm
+  loop:
+    - name: vm_enable
+      value: "YES"
+    - name: vm_dir
+      value: "zfs:{{ bhyve_dataset }}"
+    - name: vm_list
+      value: "{{ bhyve_list|community.general.json_query('[?enabled==`true`].name')|join(' ') }}"
+    - name: vm_delay
+      value: "5"  
+
+- name: init vm-bhyve
+  command: vm init
+  args:
+    creates: "{{ bhyve_mountpoint }}/.templates"
+
+- name: Install Configuration
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0644
+    owner: root
+    group: wheel
+  loop:
+    - src: arch.conf
+      dest: "{{ bhyve_mountpoint }}/.templates/arch.conf"

From 24e5456fc60005e9e12c682ec3aac2c60b83e331 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 19:33:03 -0400
Subject: [PATCH 03/10] Allow access to port 8081.

---
 ansible/roles/bhyve/files/arch.conf             | 15 ++++++++++++---
 ansible/roles/bhyve/tasks/freebsd.yaml          |  3 +--
 ansible/roles/firewall/files/odofreebsd_pf.conf |  1 +
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
index 1eec7e2..ec63222 100644
--- a/ansible/roles/bhyve/files/arch.conf
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -3,14 +3,23 @@ loader="uefi"
 graphics="yes"
 xhci_mouse="yes"
 graphics_listen="127.0.0.1"
-#graphics_listen="10.216.1.1"
+# graphics_listen="10.216.1.1"
 graphics_res="1920x1080"
-#prestart=""
+# graphics_wait="yes"
+# prestart=""
+console="tmux"
 
 cpu=1
 memory=1024M
+
+# Automatic switch ifconfig networking
+# network0_type="virtio-net"
+# network0_switch="customswitch"
+
+# Manually specify network device
 network0_type="virtio-net"
-network0_switch="customswitch"
+network0_device="tap0"
+
 disk0_type="virtio-blk"
 disk0_name="disk0"
 disk0_dev="sparse-zvol"
diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml
index 8f73216..1daff27 100644
--- a/ansible/roles/bhyve/tasks/freebsd.yaml
+++ b/ansible/roles/bhyve/tasks/freebsd.yaml
@@ -11,7 +11,6 @@
       - vm-bhyve
       - tmux # for interactive consoles
       - bhyve-firmware # For UEFI
-      # - uefi-edk2-bhyve  # Other UEFI firmware?
     state: present
 
 - name: Create zfs dataset
@@ -34,7 +33,7 @@
     - name: vm_list
       value: "{{ bhyve_list|community.general.json_query('[?enabled==`true`].name')|join(' ') }}"
     - name: vm_delay
-      value: "5"  
+      value: "5"
 
 - name: init vm-bhyve
   command: vm init
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index a672763..f82f37e 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -41,4 +41,5 @@ pass in on $ext_if proto udp to any port $udp_pass_in
 pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
 
 pass in on host_uplink0 proto udp from any to any port { 53 51820 }
+pass out on host_uplink0 proto tcp from any to any port 8081
 pass in on host_uplink1

From 0d68fac26959094871a9b3f515fe4c2d773f8093 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 20:18:21 -0400
Subject: [PATCH 04/10] Don't automatically mount the bhyve dataset.

---
 ansible/roles/bhyve/tasks/freebsd.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml
index 1daff27..67b2b24 100644
--- a/ansible/roles/bhyve/tasks/freebsd.yaml
+++ b/ansible/roles/bhyve/tasks/freebsd.yaml
@@ -19,6 +19,7 @@
     state: present
     extra_zfs_properties:
       mountpoint: "{{ bhyve_mountpoint }}"
+      canmount: "noauto"
 
 - name: Enable bhyve
   community.general.sysrc:

From d7a8dd4d6782128fba9309fa9ae112091566bcc0 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 20:30:07 -0400
Subject: [PATCH 05/10] Do not auto-mount custom datasets.

---
 ansible/roles/bhyve/tasks/freebsd.yaml | 1 +
 ansible/roles/jail/tasks/freebsd.yaml  | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml
index 67b2b24..79f02a5 100644
--- a/ansible/roles/bhyve/tasks/freebsd.yaml
+++ b/ansible/roles/bhyve/tasks/freebsd.yaml
@@ -20,6 +20,7 @@
     extra_zfs_properties:
       mountpoint: "{{ bhyve_mountpoint }}"
       canmount: "noauto"
+      "ta:bemount": "on"
 
 - name: Enable bhyve
   community.general.sysrc:
diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index e2b7e6e..39368bd 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -10,7 +10,7 @@
   zfs:
     name: "{{ item.dataset|default(jail_zfs_dataset) }}/jails/{{ item.name }}"
     state: present
-    extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({})) }}'
+    extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({}))|combine({''canmount'': ''noauto'', ''ta:bemount'': ''on''}) }}'
 
   loop: "{{ jail_list }}"
 
@@ -27,7 +27,7 @@
   zfs:
     name: "{{ item.0.dataset|default(jail_zfs_dataset) }}/persistent/{{ item.0.name }}/{{ item.1.name }}"
     state: present
-    extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({})) }}'
+    extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({}))|combine({''canmount'': ''noauto'', ''ta:bemount'': ''on''}) }}'
   loop: "{{ jail_list|subelements('persist', skip_missing=True) }}"
 
 - name: Install scripts

From 683c264650960748d253a6bfc85067136fa845d9 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 22:42:46 -0400
Subject: [PATCH 06/10] Add a script to automatically mount datasets.

---
 ansible/playbook.yaml                  |   4 +-
 ansible/roles/base/files/bemount.bash  | 126 +++++++++++++++++++++++++
 ansible/roles/base/files/bemount_rc.sh |  20 ++++
 ansible/roles/base/tasks/freebsd.yaml  |  28 ++++++
 4 files changed, 176 insertions(+), 2 deletions(-)
 create mode 100644 ansible/roles/base/files/bemount.bash
 create mode 100644 ansible/roles/base/files/bemount_rc.sh

diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index 9e387b6..858e900 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -9,7 +9,7 @@
     # - zsh
     # - network
     # - sshd
-    # - base
+    - base
     # - firewall
     # - cpu
     # - ntp
@@ -27,4 +27,4 @@
     # - fuse
     # - autofs
     # - exfat
-    - bhyve
+    # - bhyve
diff --git a/ansible/roles/base/files/bemount.bash b/ansible/roles/base/files/bemount.bash
new file mode 100644
index 0000000..84054d7
--- /dev/null
+++ b/ansible/roles/base/files/bemount.bash
@@ -0,0 +1,126 @@
+#!/usr/local/bin/bash
+#
+# Mount non-boot-environment datasets.
+#
+# We can't leave datasets outside the boot environment (for example,
+# jails or bhyve VMs) as canmount=on because then every boot
+# environment's external datasets would all attempt to mount every
+# time. To work around this, we mark those datasets as canmount=noauto
+# and run this script to mount datasets under the root of our boot
+# environment. This script depends heavily on my zfs dataset structure
+# so it needs to be improved to be robust enough for different
+# layouts. An example of my layout is:
+#
+## NAME                                                                                               MOUNTPOINT                  CANMOUNT  TA:BEMOUNT
+## zroot                                                                                              none                        off       -
+## zroot/global                                                                                       /global                     on        -
+## zroot/freebsd                                                                                      none                        on        -
+## zroot/freebsd/13.1-RELEASE                                                                         none                        on        -
+## zroot/freebsd/13.1-RELEASE/be                                                                      none                        on        -
+## zroot/freebsd/13.1-RELEASE/be/main                                                                 /                           noauto    -
+## zroot/freebsd/13.1-RELEASE/jails                                                                   none                        on        -
+## zroot/freebsd/13.1-RELEASE/jails/foo                                                               /jail/foo                   noauto    on
+## zroot/freebsd/13.1-RELEASE/jails/bar                                                               /jail/bar                   noauto    on
+## zroot/freebsd/13.1-RELEASE/jails/baz                                                               /jail/baz                   noauto    on
+## zroot/freebsd/13.1-RELEASE/vm-bhyve                                                                /vm                         noauto    on
+## zroot/linux                                                                                        none                        on        -
+## zroot/linux/arch                                                                                   none                        on        -
+## zroot/linux/arch/be                                                                                none                        on        -
+## zroot/linux/arch/be/main                                                                           /                           noauto    -
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    local all_zfs_datasets=$(zfs list -Hp -o 'name,mountpoint,canmount,ta:bemount,mounted')
+    local root_dataset=$(find_root_dataset "$all_zfs_datasets")
+    local datasets_to_mount=$(find_datasets_to_mount_for_boot_environment "$all_zfs_datasets" "$root_dataset")
+    if [ -n "$datasets_to_mount" ]; then
+        mount_datasets "$datasets_to_mount"
+    fi
+}
+
+function reverse_lines {
+    sed '1!x;H;1h;$!d;g'
+}
+
+function find_dataset {
+    local all_zfs_datasets="$1"
+    local dataset_name="$2"
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        if [ "$ds_name" = "$dataset_name" ]; then
+            echo "$dataset"
+            return
+        fi
+    done<<<"$all_zfs_datasets"
+}
+
+function find_root_dataset {
+    local all_zfs_datasets="$1"
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        local ds_mountpoint=$(awk '{print $2}'<<<"$dataset")
+        # local ds_canmount=$(awk '{print $3}'<<<"$dataset")
+        # local ds_bemount=$(awk '{print $4}'<<<"$dataset")
+        local ds_mounted=$(awk '{print $5}'<<<"$dataset")
+        if [ "$ds_mounted" = "yes" ] && [ "$ds_mountpoint" = "/" ]; then
+            echo "$ds_name"
+            return
+        fi
+    done<<<"$all_zfs_datasets"
+}
+
+function find_datasets_to_mount_for_boot_environment {
+    local all_zfs_datasets="$1"
+    local root_dataset="$2"
+    # This is a consequence of my layout for zfs datasets. I should
+    # make this more robust. Perhaps a zfs property like search up
+    # from dataset mounted at / until you find a dataset with property
+    # ta:bemountroot="on"?
+    local be_root_name="${root_dataset%/*/*}"
+    local be_root_dataset=$(find_dataset "$all_zfs_datasets" "$be_root_name")
+
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        # local ds_mountpoint=$(awk '{print $2}'<<<"$dataset")
+        local ds_canmount=$(awk '{print $3}'<<<"$dataset")
+        local ds_bemount=$(awk '{print $4}'<<<"$dataset")
+        local ds_mounted=$(awk '{print $5}'<<<"$dataset")
+
+        case "$ds_name" in
+            "${be_root_name}/"*) ;;
+            *)    continue ;;
+        esac
+
+        if [ "$ds_bemount" != "on" ]; then
+            continue
+        fi
+
+        if [ "$ds_mounted" != "no" ]; then
+            continue
+        fi
+
+        if [ "$ds_canmount" != "noauto" ]; then
+            continue
+        fi
+        echo "$dataset"
+
+    done<<<"$all_zfs_datasets"
+}
+
+function mount_datasets {
+    local datasets_to_mount=$(reverse_lines<<<"$1")
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        local ds_mountpoint=$(awk '{print $2}'<<<"$dataset")
+        local ds_canmount=$(awk '{print $3}'<<<"$dataset")
+        local ds_bemount=$(awk '{print $4}'<<<"$dataset")
+        local ds_mounted=$(awk '{print $5}'<<<"$dataset")
+
+        mount -v -t zfs "$ds_name" "$ds_mountpoint"
+
+    done<<<"$datasets_to_mount"
+}
+
+main
diff --git a/ansible/roles/base/files/bemount_rc.sh b/ansible/roles/base/files/bemount_rc.sh
new file mode 100644
index 0000000..b4eaa5c
--- /dev/null
+++ b/ansible/roles/base/files/bemount_rc.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+#
+# REQUIRE: FILESYSTEM kld
+# PROVIDE: bemount
+
+. /etc/rc.subr
+name=bemount
+rcvar=${name}_enable
+start_cmd="${name}_start"
+stop_cmd="${name}_stop"
+load_rc_config $name
+
+bemount_start() {
+    /usr/local/bin/bemount
+}
+
+bemount_stop() {
+}
+
+run_rc_command "$1"
diff --git a/ansible/roles/base/tasks/freebsd.yaml b/ansible/roles/base/tasks/freebsd.yaml
index 6d3315b..363b2f8 100644
--- a/ansible/roles/base/tasks/freebsd.yaml
+++ b/ansible/roles/base/tasks/freebsd.yaml
@@ -94,3 +94,31 @@
       src: tmpfs
       fstype: tmpfs
       opts: rw,mode=777
+
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: bemount.bash
+      dest: /usr/local/bin/bemount
+
+- name: Install rc script
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}"
+    owner: root
+    group: wheel
+    mode: 0755
+  loop:
+    - src: bemount_rc.sh
+      dest: bemount
+
+- name: Enable bemount
+  community.general.sysrc:
+    name: bemount_enable
+    value: "YES"
+    path: /etc/rc.conf.d/bemount

From b7139ded65d94a02f29cd91956bf23e9e505849c Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 23:03:30 -0400
Subject: [PATCH 07/10] Make debugging easier.

---
 ansible/playbook.yaml               | 4 ++--
 ansible/roles/bhyve/files/arch.conf | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index 858e900..9e387b6 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -9,7 +9,7 @@
     # - zsh
     # - network
     # - sshd
-    - base
+    # - base
     # - firewall
     # - cpu
     # - ntp
@@ -27,4 +27,4 @@
     # - fuse
     # - autofs
     # - exfat
-    # - bhyve
+    - bhyve
diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
index ec63222..4dd1422 100644
--- a/ansible/roles/bhyve/files/arch.conf
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -2,7 +2,8 @@
 loader="uefi"
 graphics="yes"
 xhci_mouse="yes"
-graphics_listen="127.0.0.1"
+graphics_listen="0.0.0.0"
+# graphics_listen="127.0.0.1"
 # graphics_listen="10.216.1.1"
 graphics_res="1920x1080"
 # graphics_wait="yes"
@@ -18,7 +19,7 @@ memory=1024M
 
 # Manually specify network device
 network0_type="virtio-net"
-network0_device="tap0"
+network0_device="host_link3"
 
 disk0_type="virtio-blk"
 disk0_name="disk0"

From 9bf0875ad92376e80c4c9e2a5238156e903ff604 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 23:10:49 -0400
Subject: [PATCH 08/10] Commenting out the network makes it work.

---
 ansible/roles/bhyve/files/arch.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
index 4dd1422..c015c57 100644
--- a/ansible/roles/bhyve/files/arch.conf
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -18,8 +18,8 @@ memory=1024M
 # network0_switch="customswitch"
 
 # Manually specify network device
-network0_type="virtio-net"
-network0_device="host_link3"
+# network0_type="virtio-net"
+# network0_device="host_link3"
 
 disk0_type="virtio-blk"
 disk0_name="disk0"

From 9406b8337ebbe47169d8e8cb0a852bb28f05d4f7 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 23:43:48 -0400
Subject: [PATCH 09/10] Use netgraph networking in bhyve.

---
 ansible/playbook.yaml               |  2 +-
 ansible/roles/bhyve/files/arch.conf | 11 +++--------
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index 9e387b6..2012a1b 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -23,7 +23,7 @@
     # - firefox
     # - devfs
     # - ssh_client
-    # - jail
+    - jail
     # - fuse
     # - autofs
     # - exfat
diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
index c015c57..ed067e2 100644
--- a/ansible/roles/bhyve/files/arch.conf
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -13,15 +13,10 @@ console="tmux"
 cpu=1
 memory=1024M
 
-# Automatic switch ifconfig networking
-# network0_type="virtio-net"
-# network0_switch="customswitch"
-
-# Manually specify network device
-# network0_type="virtio-net"
-# network0_device="host_link3"
-
 disk0_type="virtio-blk"
 disk0_name="disk0"
 disk0_dev="sparse-zvol"
 virt_random="yes"  # virtio-rnd
+
+# Creates a link to host_bridge1's link3 hook to the vmlink hook on a type socket
+bhyve_options="-s 2:0,virtio-net,netgraph,path=host_bridge1:,peerhook=link3"

From 0f8ebc5bbe55974c62cc635c42200b9f54aea043 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 1 Nov 2022 23:50:53 -0400
Subject: [PATCH 10/10] cleanup

---
 ansible/playbook.yaml               | 48 ++++++++++++++---------------
 ansible/roles/bhyve/files/arch.conf |  4 +--
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index 2012a1b..716378b 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -2,29 +2,29 @@
   vars:
     ansible_become: True
   roles:
-    # - sudo
-    # - users
-    # - package_manager
-    # - zrepl
-    # - zsh
-    # - network
-    # - sshd
-    # - base
-    # - firewall
-    # - cpu
-    # - ntp
-    # - build
-    # - graphics
-    # - gpg
-    # - fonts
-    # - alacritty
-    # - sway
-    # - emacs
-    # - firefox
-    # - devfs
-    # - ssh_client
+    - sudo
+    - users
+    - package_manager
+    - zrepl
+    - zsh
+    - network
+    - sshd
+    - base
+    - firewall
+    - cpu
+    - ntp
+    - build
+    - graphics
+    - gpg
+    - fonts
+    - alacritty
+    - sway
+    - emacs
+    - firefox
+    - devfs
+    - ssh_client
     - jail
-    # - fuse
-    # - autofs
-    # - exfat
+    - fuse
+    - autofs
+    - exfat
     - bhyve
diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf
index ed067e2..ef6ff6d 100644
--- a/ansible/roles/bhyve/files/arch.conf
+++ b/ansible/roles/bhyve/files/arch.conf
@@ -2,8 +2,8 @@
 loader="uefi"
 graphics="yes"
 xhci_mouse="yes"
-graphics_listen="0.0.0.0"
-# graphics_listen="127.0.0.1"
+# graphics_listen="0.0.0.0"
+graphics_listen="127.0.0.1"
 # graphics_listen="10.216.1.1"
 graphics_res="1920x1080"
 # graphics_wait="yes"