Add ndproxy to support neighbor discovery of VMs from the public internet.

Currently, I get a lot of dropped packets on 6ghz. I suspect it is a bug in the drivers/firmware.

ansible/environments/colo/host_vars/mrmanager

@@ -14,6 +14,8 @@ pf_config: "mrmanager_pf.conf"
 pflog_conf:
   - name: 0
     dev: pflog0
+  - name: 1
+    dev: pflog1
 cputype: "amd"
 hwpstate: true
 etc_hosts: {}
@@ -36,6 +38,10 @@ jail_list:
     enabled: true
     conf:
       src: public_dns
+  - name: rg
+    enabled: true
+    conf:
+      src: rg
 bhyve_dataset: zdata/vm
 bhyve_canmount: "on"
 # efi_dev: /dev/gpt/EFI

ansible/environments/home/host_vars/homeserver

@@ -4,6 +4,7 @@ pkgbase_url: "https://freebsdpkg.fizz.buzz/pkgbase/14broadwell-repo/FreeBSD:14:a
 zfs_snapshot_datasets:
   - path: zroot/freebsd/computer/be
   - path: zmass/encrypted/vm
+  - path: zmass/encrypted/data
 users:
   talexander:
     initialize: true
@@ -24,6 +25,8 @@ users:
     gitconfig: "gitconfig_home"
 sshd_enabled: true
 sshd_conf: "sshd_config"
+prefer_ipv6: true
+dummynet_config: "dnctl.conf"
 pf_config: "homeserver_pf.conf"
 pflog_conf:
   - name: 0
@@ -64,6 +67,9 @@ jail_list:
   - name: certificate
     conf:
       src: certificate
+  - name: momlaptop
+    conf:
+      src: momlaptop
   # - name: mumble
   #   conf:
   #     src: mumble
@@ -71,9 +77,17 @@ jail_list:
   #     - name: mumbledb
   #       mount: /var/db/murmur
 bhyve_dataset: zmass/encrypted/vm
-bhyve_list: []
-bhyve_canmount: "on"
+# Disable mounting bhyve dataset so it doesn't hide the unencrypted linfi vm
+bhyve_canmount: "off"
+bhyve_mountpoint: "none"
 bhyve_bemount: "on"
 wireguard_directory: homeserver
 enabled_wireguard:
   - wgh
+linfi:
+  enabled: true
+  zfs_dataset: zmass/unencrypted/vm/linfi
+  zfs_mountpoint: /vm/linfi
+  driver_blocklist: "ath if_ath if_ath_pci ath_hal if_iwm if_iwlwifi"
+  pci_blocklist: "6/0/0"
+  amd: false

ansible/environments/home/hosts

@@ -1,2 +1,2 @@
 [headless]
-homeserver ansible_user=talexander ansible_host=10.216.1.1
+homeserver ansible_user=talexander ansible_host=homeserver

ansible/environments/jail/host_vars/momlaptop

@@ -0,0 +1 @@
+os_flavor: freebsd

ansible/environments/jail/hosts

@@ -8,3 +8,4 @@ public_dns ansible_ssh_host=public_dns@10.217.2.1 ansible_connection=sshjail
 sftp ansible_ssh_host=sftp@homeserver ansible_connection=sshjail
 bastion ansible_ssh_host=bastion@homeserver ansible_connection=sshjail
 certificate ansible_ssh_host=certificate@homeserver ansible_connection=sshjail
+momlaptop ansible_ssh_host=momlaptop@homeserver ansible_connection=sshjail

ansible/environments/laptop/host_vars/odofreebsd

@@ -5,10 +5,12 @@ zfs_snapshot_datasets:
   - path: zroot/freebsd/current/be/default
 sshd_enabled: true
 sshd_conf: "sshd_config"
-#pf_config: "odofreebsd_pf.conf"
-#pflog_conf:
-#  - name: 0
-#    dev: pflog0
+pf_config: "odofreebsd_pf.conf"
+pflog_conf:
+ - name: 0
+   dev: pflog0
+prefer_ipv6: true
+dummynet_config: "dnctl.conf"
 network_rc: "odofreebsd_network.conf"
 rc_conf: "odofreebsd_rc.conf"
 loader_conf: "odofreebsd_loader.conf"
@@ -40,13 +42,14 @@ users:
 devfs_rules: "odo_devfs.rules"
 jail_zfs_dataset: zroot/freebsd/current/jails
 jail_zfs_dataset_mountpoint: /jail
+jail_canmount: "on"
 jail_list:
   - name: nat_dhcp
     enabled: true
     conf:
       src: nat_dhcp
 bhyve_dataset: zroot/freebsd/current/vm
-bhyve_list: []
+bhyve_bemount: off
 # efi_dev: /dev/gpt/EFI
 efi_dev: /dev/diskid/DISK-SJB7N717610407Q0Hp1
 sway_conf_files:
@@ -56,3 +59,10 @@ enabled_wireguard:
   - wgh
   - drmario
   - colo
+linfi:
+  enabled: true
+  zfs_dataset: zroot/freebsd/current/vm/linfi
+  zfs_mountpoint: /vm/linfi
+  driver_blocklist: "if_iwm if_iwlwifi"
+  pci_blocklist: "1/0/0"
+  amd: true

ansible/environments/vm/host_vars/poudrieremrmanager

@@ -1,4 +1,5 @@
 os_flavor: "freebsd"
+sshd_enabled: true
 custom_repo: "file:///usr/local/poudriere/data/packages/currentznver4-default-framework"
 pkgbase_url: "file:///usr/local/poudriere/data/images/currentznver4-repo/FreeBSD:15:amd64/latest"
 poudriere_builds:

ansible/playbook.yaml

@@ -27,6 +27,7 @@
     - sway
     - emacs
     - firefox
+    - chromium
     - devfs
     - ssh_client
     - sshfs
@@ -52,7 +53,7 @@
     - javascript
     - launch_keyboard
     - lvfs
-    - restaurant_health_rating
+    # - restaurant_health_rating
     - wasm
     - noise_suppression
 
@@ -67,9 +68,12 @@
     ansible_become: True
   roles:
     - sudo # for poudboot script
+    - doas
     - fstab
     - package_manager
+    - zsh
     - termcap
+    - sshd
     - portshaker
     - poudriere
     - poudrierenginx
@@ -100,6 +104,7 @@
     - wireguard
     - emacs
     - mrmanager
+    - ndproxy
 
 - hosts: admin_git:public_dns
   vars:
@@ -122,12 +127,14 @@
   vars:
     ansible_become: True
   roles:
+    - linfi
     - framework_laptop
 
 - hosts: homeserver
   vars:
     ansible_become: True
   roles:
+    - linfi
     - homeserver
 
 - hosts: odowork
@@ -154,3 +161,9 @@
     ansible_become: True
   roles:
     - jail_certificate
+
+- hosts: momlaptop
+  vars:
+    ansible_become: True
+  roles:
+    - jail_momlaptop

ansible/roles/base/files/bbr_loader.conf

@@ -0,0 +1 @@
+tcp_bbr_load="YES"

ansible/roles/base/files/gitconfig_home

@@ -1,35 +1,54 @@
 [user]
 	email = tom@fizz.buzz
 	name = Tom Alexander
-	signingkey = D3A179C9A53C0EDE
+	signingkey = 36C99E8B3C39D85F
 [push]
-	default = simple
+	default = simple # (default since 2.0)
 [alias]
 	lg = log --color --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit
 	bh = log --oneline --branches=* --remotes=* --graph --decorate
 	amend = commit --amend --no-edit
+        authorcount = shortlog --summary --numbered --all --no-merges
 [core]
 	excludesfile = ~/.gitignore_global
 [commit]
 	gpgsign = true
+	verbose = true
 [pull]
 	rebase = true
 [log]
 	date = local
 [init]
 	defaultBranch = main
-
-# Use meld for `git difftool` and `git mergetool`
 [diff]
-	tool = meld
+	tool = meld # Use meld for `git difftool` and `git mergetool`
+	algorithm = histogram
+	colorMoved = plain
+	mnemonicPrefix = true
+	renames = true
 [difftool]
 	prompt = false
 [difftool "meld"]
 	cmd = meld "$LOCAL" "$REMOTE"
 [merge]
 	tool = meld
+	conflictStyle = zdiff3
 [mergetool "meld"]
         # Make the middle pane start with partially-merged contents:
 	cmd = meld "$LOCAL" "$MERGED" "$REMOTE" --output "$MERGED"
         # Make the middle pane start without any merge progress:
 	# cmd = meld "$LOCAL" "$BASE" "$REMOTE" --output "$MERGED"
+[column]
+	ui = auto
+[branch]
+	sort = -committerdate
+[tag]
+	sort = version:refname
+[fetch]
+	prune = true
+	pruneTags = true
+	all = true
+[rebase]
+	autoSquash = true
+	autoStash = true
+	updateRefs = false

ansible/roles/base/files/gitconfig_work

@@ -1,33 +1,38 @@
 [user]
 	email = ThomasA.Alexander@hmhn.org
 	name = Tom Alexander
-	signingkey = D3A179C9A53C0EDE
+	signingkey = 36C99E8B3C39D85F
 [push]
-	default = simple
+	default = simple # (default since 2.0)
 [alias]
 	lg = log --color --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit
 	bh = log --oneline --branches=* --remotes=* --graph --decorate
 	amend = commit --amend --no-edit
+        authorcount = shortlog --summary --numbered --all --no-merges
 [core]
 	excludesfile = ~/.gitignore_global
 [commit]
 	gpgsign = true
+	verbose = true
 [pull]
 	rebase = true
 [log]
 	date = local
 [init]
 	defaultBranch = main
-
-# Use meld for `git difftool` and `git mergetool`
 [diff]
-	tool = meld
+	tool = meld # Use meld for `git difftool` and `git mergetool`
+	algorithm = histogram
+	colorMoved = plain
+	mnemonicPrefix = true
+	renames = true
 [difftool]
 	prompt = false
 [difftool "meld"]
 	cmd = meld "$LOCAL" "$REMOTE"
 [merge]
 	tool = meld
+	conflictStyle = zdiff3
 [mergetool "meld"]
         # Make the middle pane start with partially-merged contents:
 	cmd = meld "$LOCAL" "$MERGED" "$REMOTE" --output "$MERGED"
@@ -35,3 +40,19 @@
 	# cmd = meld "$LOCAL" "$BASE" "$REMOTE" --output "$MERGED"
 [includeIf "gitdir:/bridge/"]
 	path = /bridge/git/machine_setup/ansible/roles/base/files/gitconfig_home
+[includeIf "gitdir:/persist/"]
+	path = /bridge/git/machine_setup/ansible/roles/base/files/gitconfig_home
+[column]
+	ui = auto
+[branch]
+	sort = -committerdate
+[tag]
+	sort = version:refname
+[fetch]
+	prune = true
+	pruneTags = true
+	all = true
+[rebase]
+	autoSquash = true
+	autoStash = true
+	updateRefs = false

ansible/roles/base/files/login.conf

@@ -44,6 +44,7 @@ default:\
 	:pseudoterminals=unlimited:\
 	:kqueues=unlimited:\
 	:umtxp=unlimited:\
+	:pipebuf=unlimited:\
 	:priority=0:\
 	:ignoretime@:\
 	:umask=022:\

ansible/roles/base/files/watch_freebsd

@@ -10,7 +10,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 function cleanup {
     switch_to_main_screen
 }
-for sig in EXIT INT QUIT HUP TERM; do
+for sig in EXIT; do
   trap "set +e; cleanup; exit" "$sig"
 done
 

ansible/roles/base/tasks/common.yaml

@@ -19,6 +19,7 @@
       - tcpdump
       - moreutils # for ts [%Y-%m-%d %H:%M:%.S]
       - ddrescue
+      - dmidecode
     state: present
 
 - name: Install packages

ansible/roles/base/tasks/freebsd.yaml

@@ -13,6 +13,7 @@
       - gsed
       - gmake
       - rust-coreutils
+      - shuf
     state: present
 
 - name: Install service configuration
@@ -38,18 +39,6 @@
   command: cap_mkdb /etc/login.conf
   when: login_config.changed
 
-- name: Enable periodic scrub
-  community.general.sysrc:
-    name: daily_scrub_zfs_enable
-    value: "YES"
-    path: /etc/periodic.conf.local
-
-- name: Set scrub interval
-  community.general.sysrc:
-    name: daily_scrub_zfs_default_threshold
-    value: "7"
-    path: /etc/periodic.conf.local
-
 - name: Install loader.conf
   copy:
     src: "{{loader_conf}}"
@@ -119,3 +108,65 @@
     group: wheel
   loop:
     - disk_labels
+
+- name: Configure sysctls
+  sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: false
+    sysctl_file: "/etc/sysctl.conf.local"
+  loop:
+    # Adjust ttl
+    - name: net.inet.ip.ttl
+      value: 65
+    - name: net.inet6.ip6.hlim
+      value: 65
+
+- name: Log periodic output instead of getting it as mail
+  blockinfile:
+    path: "/etc/periodic.conf.local"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK log"
+    # create: true
+    mode: 0644
+    owner: root
+    group: wheel
+    block: |
+      daily_output=/var/log/daily.log
+      weekly_output=/var/log/weekly.log
+      monthly_output=/var/log/monthly.log
+
+- name: Enable periodic zfs scrub
+  when: install_zfs
+  blockinfile:
+    path: "/etc/periodic.conf.local"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK zfs"
+    # create: true
+    mode: 0644
+    owner: root
+    group: wheel
+    block: |
+      daily_scrub_zfs_enable="YES"
+      daily_scrub_zfs_default_threshold="7"
+
+# Switch to bbr tcp congestion control which should be better on lossy connections like bad wifi.
+- name: Install loader.conf
+  copy:
+    src: "files/{{ item }}_loader.conf"
+    dest: "/boot/loader.conf.d/{{ item }}.conf"
+    mode: 0644
+    owner: root
+    group: wheel
+  loop:
+    - bbr
+
+- name: Configure sysctls
+  sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: false
+    sysctl_file: "/etc/sysctl.conf.local"
+  loop:
+    - name: net.inet.tcp.functions_default
+      value: "bbr"

ansible/roles/base/tasks/linux.yaml

@@ -67,3 +67,13 @@
     - name: vm.dirty_writeback_centisecs
       value: 1500
       file: power.conf
+    # Adjust ttl
+    - name: net.ipv4.ip_default_ttl
+      value: 65
+      file: ttl.conf
+    - name: net.ipv6.conf.all.hop_limit
+      value: 65
+      file: ttl.conf
+    - name: net.ipv6.conf.default.hop_limit
+      value: 65
+      file: ttl.conf

ansible/roles/bhyve/defaults/main.yaml

@@ -1,2 +1 @@
 bhyve_mountpoint: "/vm"
-bhyve_list: []

ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash

@@ -30,6 +30,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 : ${BRIDGE_NAME:="bridge_$INTERFACE_NAME"} # or bridge_raw for RAW networks
 : ${VNC_ENABLE:="NO"}
 : ${VNC_LISTEN:="127.0.0.1:5900"}
+: ${VNC_WIDTH:="1920"}
+: ${VNC_HEIGHT:="1080"}
 
 if [ "$VERBOSE" = "YES" ]; then
     set -x
@@ -45,7 +47,7 @@ function cleanup {
     done
 }
 vms=()
-for sig in EXIT INT QUIT HUP TERM; do
+for sig in EXIT; do
   trap "set +e; sleep 10; cleanup" "$sig"
 done
 
@@ -105,7 +107,8 @@ function start_vm {
     local bridge_name="$BRIDGE_NAME"
     local ip_range="$IP_RANGE" # for raw this value does not matter
 
-    local mac_address=$(calculate_mac_address "$name")
+    local mac_address
+    mac_address=$(calculate_mac_address "$name")
 
     local additional_args=()
 
@@ -140,7 +143,7 @@ function start_vm {
         additional_args+=("-s" "5,ahci-cd,$mount_cd")
     fi
     if [ "$VNC_ENABLE" = "YES" ]; then
-        additional_args+=("-s" "29,fbuf,tcp=$VNC_LISTEN,w=1920,h=1080")
+        additional_args+=("-s" "29,fbuf,tcp=$VNC_LISTEN,w=$VNC_WIDTH,h=$VNC_HEIGHT")
     fi
     vms+=("$name")
     while true; do
@@ -150,7 +153,10 @@ function start_vm {
             -D \
             -c $CPU_CORES \
             -m $MEMORY \
+            -S \
             -H \
+            -P \
+            -o 'rtc.use_localtime=false' \
             -s 0,hostbridge \
             -s "4,nvme,/dev/zvol/${zfs_path}/disk0" \
             -s 30,xhci,tablet \
@@ -211,7 +217,7 @@ EOF
 mkpeer ${host_interface_name}: bridge ether link0
 name ${host_interface_name}:ether $bridge_name
 EOF
-        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+        ifconfig "$(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2)" name "${host_interface_name}" "$ip_range" up
     fi
 }
 
@@ -245,7 +251,8 @@ function ng_exists {
 
 function calculate_mac_address {
     local name="$1"
-    local source=$(md5 -r -s "$name" | awk '{print $1}')
+    local source
+    source=$(md5 -r -s "$name" | awk '{print $1}')
     echo "06:${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}"
 }
 

ansible/roles/bhyve/files/bhyverc.bash

@@ -0,0 +1,478 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Share a host directory to the guest via 9pfs.
+#
+# Inside the VM run:
+#   mount -t virtfs -o trans=virtio sharename /some/vm/path
+#   mount -t 9p -o cache=mmap -o msize=512000 sharename /mnt/9p
+#   mount -t 9p -o trans=virtio,cache=mmap,msize=512000 bind9p /path/to/mountpoint
+# bhyve_options="-s 28,virtio-9p,sharename=/"
+
+# Enable Sound
+# bhyve_options="-s 16,hda,play=/dev/dsp,rec=/dev/dsp"
+
+# Example usage:
+#
+#   doas bhyverc create-disk zdata/vm/poudriere /vm/poudriere 10
+#   doas bhyverc start poudriere zdata/vm/poudriere /vm/poudriere /vm/iso/FreeBSD-13.2-RELEASE-amd64-bootonly.iso
+#   doas bhyverc start poudriere zdata/vm/poudriere /vm/poudriere
+
+
+: ${VERBOSE:="NO"} # or YES
+if [ "$VERBOSE" = "YES" ]; then
+    set -x
+fi
+
+: ${CPU_CORES:="1"}
+: ${MEMORY:="1G"}
+: ${NETWORK:="NAT"} # or RAW or BOTH
+: ${IP_RANGE:="10.215.1.1/24"} # Ignored for RAW networks
+: ${INTERFACE_NAME:="jail_nat"} # or the external interface like lagg0 for RAW networks
+: ${BRIDGE_NAME:="bridge_$INTERFACE_NAME"} # or bridge_raw for RAW networks
+: ${VNC_ENABLE:="NO"}
+: ${VNC_LISTEN:="127.0.0.1:5900"}
+: ${VNC_WIDTH:="1920"}
+: ${VNC_HEIGHT:="1080"}
+: ${BIND9P:=""}
+: ${PREVENT_OOM:="NO"}
+: "${CD:=}"
+
+: ${SHUTDOWN_TIMEOUT:="600"} # 10 minutes
+
+
+
+############## Setup #########################
+
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function log {
+    (>&2 echo "${@}")
+}
+
+############## Program #########################
+
+function main {
+    local cmd
+    cmd=$1
+    shift
+    if [ "$cmd" = "start" ]; then
+        init
+        start "${@}"
+    elif [ "$cmd" = "stop" ]; then
+        init
+        stop "${@}"
+    elif [ "$cmd" = "status" ]; then
+        init
+        status "${@}"
+    elif [ "$cmd" = "console" ]; then
+        init
+        console "${@}"
+    elif [ "$cmd" = "_start_body" ]; then
+        init
+        start_body "${@}"
+    elif [ "$cmd" = "create-disk" ]; then
+        create_disk "${@}"
+    else
+        (>&2 echo "Unknown command: $cmd")
+        exit 1
+    fi
+}
+
+function start {
+    local num_vms="$#"
+    if [ "$num_vms" -eq 0 ]; then
+        log "No VMs specified."
+        return 0
+    fi
+
+    while [ "$#" -gt 0 ]; do
+        local name="$1"
+        shift 1
+        log "Starting VM $name."
+        start_one "$name"
+        [ "$#" -eq 0 ] || sleep 5
+    done
+}
+
+function start_one {
+    local name="$1"
+    local tmux_name="$name"
+    /usr/local/bin/tmux new-session -d -s "$tmux_name" "$0" "_start_body" "$name"
+    # /usr/local/bin/tmux new-session -d -s "$tmux_name" "/usr/bin/env VNC_ENABLE=NO VNC_LISTEN=0.0.0.0:5900 /usr/local/bin/bash /home/talexander/launch_opnsense.bash"
+}
+
+function launch_pidfile {
+    local pidfile="$1"
+    shift 1
+    mkdir -p "$(dirname "$pidfile")"
+    cat > "${pidfile}" <<< "$$"
+    set -x
+    exec "${@}"
+}
+export -f launch_pidfile
+
+function stop {
+    local num_vms="$#"
+    if [ "$num_vms" -eq 0 ]; then
+        log "No VMs specified."
+        return 0
+    fi
+
+    while [ "$#" -gt 0 ]; do
+        local name="$1"
+        shift 1
+        log "Stopping VM $name."
+        stop_one "$name"
+        [ "$#" -eq 0 ] || sleep 5
+    done
+}
+
+function stop_one {
+    local name="$1"
+    local pidfile="/run/bhyverc/${name}/pid"
+
+    if [ ! -e "$pidfile" ]; then
+        log "Pid file $pidfile does not exist."
+        return 0
+    fi
+
+    local bhyve_pid
+    bhyve_pid=$(cat "$pidfile")
+
+    if ps -p "$bhyve_pid" >/dev/null; then
+        # Send ACPI shutdown command
+        log "Sending ACPI shutdown to ${name}:${bhyve_pid}."
+        kill -SIGTERM "$bhyve_pid"
+    fi
+
+    local timeout_start timeout_end
+    timeout_start=$(date +%s)
+    while ps -p "$bhyve_pid" >/dev/null; do
+        timeout_end=$(date +%s)
+        if [ $((timeout_end-timeout_start)) -ge "$SHUTDOWN_TIMEOUT" ]; then
+            log "${name}:${bhyve_pid} took more than $SHUTDOWN_TIMEOUT seconds to shut down. Hard powering down."
+            break
+        fi
+
+        log "Waiting for ${name}:${bhyve_pid} to exit."
+        sleep 2
+    done
+
+    bhyvectl "--vm=$name" --destroy || true
+
+    local timeout_start timeout_end
+    timeout_start=$(date +%s)
+    while ps -p "$bhyve_pid" >/dev/null; do
+        timeout_end=$(date +%s)
+        if [ $((timeout_end-timeout_start)) -ge "$SHUTDOWN_TIMEOUT" ]; then
+            log "${name}:${bhyve_pid} took more than $SHUTDOWN_TIMEOUT seconds to hard power down. Giving up."
+            break
+        fi
+
+        log "Waiting for ${name}:${bhyve_pid} to hard power down."
+        sleep 2
+    done
+
+    rm -f "$pidfile"
+
+    log "Finished stopping $name."
+}
+
+function status {
+    local num_vms="$#"
+
+    if [ "$num_vms" -gt 0 ]; then
+        for name in "$@"; do
+            status_one "$name"
+        done
+    else
+        log "No VMs specified."
+    fi
+}
+
+function status_one {
+    local name="$1"
+    local pidfile="/run/bhyverc/${name}/pid"
+
+    if [ ! -e "$pidfile" ]; then
+        log "$name is not running."
+        return 0
+    fi
+
+    local bhyve_pid
+    bhyve_pid=$(cat "$pidfile")
+
+    if ! ps -p "$bhyve_pid" >/dev/null; then
+        log "$name is not running."
+        return 0
+    fi
+
+    log "$name is running as pid $bhyve_pid."
+}
+
+function console {
+    local num_vms="$#"
+
+    if [ "$num_vms" -gt 0 ]; then
+        for name in "$@"; do
+            log "Attaching to console of VM $name."
+            console_one "$name"
+        done
+    else
+        log "No VMs specified."
+    fi
+}
+
+function console_one {
+    local name="$1"
+    local tmux_name="$name"
+    exec tmux a -t "$tmux_name"
+}
+
+function init {
+    mkdir -p /run/bhyverc
+}
+
+############## Bhyve ###########################
+
+function create_disk {
+    local zfs_path="$1"
+    local mount_path="$2"
+    local gigabytes="$3"
+    zfs create -o "mountpoint=$mount_path" "$zfs_path"
+    cp /usr/local/share/edk2-bhyve/BHYVE_UEFI_VARS.fd "${mount_path}/"
+    tee "${mount_path}/settings" <<EOF
+CPU_CORES="$CPU_CORES"
+MEMORY="$MEMORY"
+NETWORK="$NETWORK"
+IP_RANGE="$IP_RANGE"
+BRIDGE_NAME="$BRIDGE_NAME"
+INTERFACE_NAME="$INTERFACE_NAME"
+EOF
+    zfs create -s "-V${gigabytes}G" -o volmode=dev -o primarycache=metadata -o secondarycache=none "$zfs_path/disk0"
+}
+
+function start_body {
+    local name="$1"
+    local zfs_path="zdata/vm/$name"
+    local mount_path="/vm/$name"
+
+    if [ -e "${mount_path}/settings" ]; then
+        source "${mount_path}/settings"
+    fi
+
+    local mount_cd="$CD"
+
+    local host_interface_name="$INTERFACE_NAME" # for raw, external interface
+    local bridge_name="$BRIDGE_NAME"
+    local ip_range="$IP_RANGE" # for raw this value does not matter
+
+    local mac_address
+    mac_address=$(calculate_mac_address "$name")
+
+    if [ "$PREVENT_OOM" = "YES" ]; then
+        protect -d -i -p "$$"
+    fi
+
+    local entry parsed_item
+    local additional_args=()
+    local next_pcie_slot=10
+
+    if [ "$NETWORK" = "NAT" ]; then
+        assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+    elif [ "$NETWORK" = "RAW" ]; then
+        assert_raw "$host_interface_name" "$bridge_name"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+    elif [ "$NETWORK" = "BOTH" ]; then
+        assert_bridge "jail_nat" "$bridge_name" "$ip_range"
+        assert_raw "$host_interface_name" "bridge_raw"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        local raw_bridge_link_name=$(detect_available_link "bridge_raw")
+        local raw_mac_address=$(calculate_mac_address "${name}_raw")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+        additional_args+=("-s" "3:0,virtio-net,netgraph,path=bridge_raw:,peerhook=${raw_bridge_link_name},mac=${raw_mac_address}")
+    else
+        die 1 "Unrecognized NETWORK type $NETWORK"
+    fi
+
+    if [ -n "$BIND9P" ]; then
+        if [[ "$BIND9P" = *":"* ]]; then
+            IFS=':' read -ra entry <<<"$BIND9P"
+            for item in "${entry[@]}"; do
+                IFS='=' read -ra parsed_item <<<"$item"
+                additional_args+=("-s" "${next_pcie_slot},virtio-9p,${parsed_item[0]}=${parsed_item[1]}")
+                next_pcie_slot=$((next_pcie_slot+1))
+            done
+        else
+            additional_args+=("-s" "${next_pcie_slot},virtio-9p,bind9p=${BIND9P}")
+            next_pcie_slot=$((next_pcie_slot+1))
+        fi
+    fi
+
+
+    # -H release the CPU when guest issues HLT instruction. Otherwise 100% of core will be consumed.
+         # -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
+             # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
+            # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080 \
+
+    # TODO: Look into using nmdm instead of stdio for serial console
+    if [ -n "$mount_cd" ]; then
+        additional_args+=("-s" "5,ahci-cd,$mount_cd")
+    fi
+    if [ "$VNC_ENABLE" = "YES" ]; then
+        additional_args+=("-s" "${next_pcie_slot},fbuf,tcp=$VNC_LISTEN,w=$VNC_WIDTH,h=$VNC_HEIGHT")
+        next_pcie_slot=$((next_pcie_slot+1))
+    fi
+    vms+=("$name")
+    while true; do
+        local pidfile="/run/bhyverc/${name}/pid"
+        trap "set +e; stop_one '${name}'" EXIT
+
+        local launch_cmd=()
+        launch_cmd+=(
+            launch_pidfile "$pidfile"
+            bhyve
+            -D
+            -c "$CPU_CORES"
+            -m "$MEMORY"
+            -S
+            -H
+            -o 'rtc.use_localtime=false'
+            -s "0,hostbridge"
+            -s "4,nvme,/dev/zvol/${zfs_path}/disk0"
+            -s "${next_pcie_slot},xhci,tablet"
+            -s "$((next_pcie_slot+1)),lpc" -l "com1,stdio"
+            -l "bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,${mount_path}/BHYVE_UEFI_VARS.fd"
+            "${additional_args[@]}"
+            "$name"
+        )
+        set +e
+        rm -f "$pidfile"
+        (
+            IFS=$' \n\t'
+            set -ex
+            bash -c "${launch_cmd[*]}"
+        )
+        local exit_code=$?
+        log "Exit code ${exit_code}"
+        set -e
+        if [ $exit_code -eq 0 ]; then
+            echo "Rebooting."
+            sleep 5
+        elif [ $exit_code -eq 1 ]; then
+            echo "Powered off."
+            break
+        elif [ $exit_code -eq 2 ]; then
+            echo "Halted."
+            break
+        elif [ $exit_code -eq 3 ]; then
+            echo "Triple fault."
+            break
+        elif [ $exit_code -eq 4 ]; then
+            echo "Exited due to an error."
+            break
+        fi
+    done
+}
+
+function detect_available_link {
+    local bridge_name="$1"
+    local linknum=1
+    while true; do
+        local link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function assert_bridge {
+    local host_interface_name="$1"
+    local bridge_name="$2"
+    local ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig "$(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2)" name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function assert_raw {
+    local extif="$1"
+    local bridge_name="$2"
+
+    kldload -n ng_bridge ng_eiface ng_ether
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctlcat <<EOF
+# Create a bridge.
+mkpeer $extif: bridge lower link0
+# Assign a name to the bridge.
+name $extif:lower ${bridge_name}
+# Since the host is also using $extif, we need to connect the upper hook also. Otherwise we will lose connectivity.
+connect $extif: ${bridge_name}: upper link1
+
+# Enable promiscuous mode so the host ethernet adapter accepts packets for all addresses
+msg $extif: setpromisc 1
+
+# Do not overwrite source address on packets
+msg $extif: setautosrc 0
+EOF
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+function calculate_mac_address {
+    local name="$1"
+    local source
+    source=$(md5 -r -s "$name" | awk '{print $1}')
+    echo "06:${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}"
+}
+
+function find_available_port {
+    local start_port="$1"
+    local port="$start_port"
+    while true; do
+        sockstat -P tcp -p 443
+        port=$((port + 1))
+    done
+}
+
+function ngctlcat {
+    if [ "$VERBOSE" = "YES" ]; then
+        tee /dev/tty | ngctl -d -f -
+    else
+        ngctl -d -f -
+    fi
+}
+
+main "${@}"

ansible/roles/bhyve/files/bhyverc.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+#
+# REQUIRE: LOGIN FILESYSTEMS
+# PROVIDE: bhyverc
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+name=bhyverc
+rcvar=${name}_enable
+start_cmd="${name}_start"
+stop_cmd="${name}_stop"
+status_cmd="${name}_status"
+console_cmd="${name}_console"
+extra_commands="console"
+load_rc_config $name
+
+bhyverc_start() {
+    export PATH="$PATH:/usr/local/bin"
+    exec /usr/local/bin/bhyverc start "${@}"
+}
+
+bhyverc_status() {
+    export PATH="$PATH:/usr/local/bin"
+    exec /usr/local/bin/bhyverc status "${@}"
+}
+
+bhyverc_stop() {
+    export PATH="$PATH:/usr/local/bin"
+    exec /usr/local/bin/bhyverc stop "${@}"
+}
+
+bhyverc_console() {
+    export PATH="$PATH:/usr/local/bin"
+    exec /usr/local/bin/bhyverc console "${@}"
+}
+
+run_rc_command "$@"

ansible/roles/bhyve/tasks/freebsd.yaml

@@ -22,6 +22,25 @@
   loop:
     - src: bhyve_netgraph_bridge.bash
       dest: /usr/local/bin/bhyve_netgraph_bridge
+    - src: bhyverc.bash
+      dest: /usr/local/bin/bhyverc
+
+- name: Install rc script
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}"
+    owner: root
+    group: wheel
+    mode: 0755
+  loop:
+    - src: bhyverc.sh
+      dest: bhyverc
+
+- name: Enable bhyverc
+  community.general.sysrc:
+    name: bhyverc_enable
+    value: "YES"
+    path: /etc/rc.conf.d/bhyverc
 
 - name: Create zfs dataset
   zfs:

ansible/roles/build/files/aurutils-sync

@@ -5,4 +5,4 @@ set -euo pipefail
 IFS=$'\n\t'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-GPGKEY=27DE40D9B8455C1B exec aur sync --makepkg-conf /etc/aurutils/makepkg.conf -c --sign "$@"
+GPGKEY=4278299FB84F6875 exec aur sync --makepkg-conf /etc/aurutils/makepkg.conf -c --sign "$@"

ansible/roles/build/files/gpg.asc

@@ -1,27 +1,27 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
-mDMEXZwWGhYJKwYBBAHaRw8BAQdAfv7qozKkmf4D+5PDzADsMm4aAKDGLha7+Cu0
-0H+RsWG0HlRvbSBBbGV4YW5kZXIgPHdvcmtAZml6ei5idXp6PoiQBBMWCAA4FiEE
-uEgVk2PCh3kXlUvhJ95A2bhFXBsFAl+w+R0CGwMFCwkIBwIGFQoJCAsCBBYCAwEC
-HgECF4AACgkQJ95A2bhFXBt6fgD+NOYnw9gz5K/q3H5LE/JvqzCSHezJmeGgif0C
-uU4m1/MA+gPDKME7syEtJsTpELEMrxWWpDW0tD/W1iJE7roGYPQPtB1Ub20gQWxl
-eGFuZGVyIDx0b21AZml6ei5idXp6PoiQBBMWCAA4FiEEuEgVk2PCh3kXlUvhJ95A
-2bhFXBsFAl2cFhoCGwMFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQJ95A2bhF
-XBvYJQEA19wc2s/bEKcnHONC3i8UikLFqZXyYoH6/MFjoAteU8sBAKpE7Qq0zbJb
-XWRESzK3u6p7/+kUqOeDltAuKXTe1FAGuDMEXZwWyhYJKwYBBAHaRw8BAQdAPyIL
-4EGg4T5JO9q2kpVDy2WjMiXz3nZXwYW4GLoTYkiI9QQYFggAJgIbAhYhBLhIFZNj
-wod5F5VL4SfeQNm4RVwbBQJlC4ZhBQkLMdaXAIF2IAQZFggAHRYhBIHmRDmWdVAu
-sSUWutOhecmlPA7eBQJdnBbKAAoJENOhecmlPA7ejJ4A/iq7N2mMhx+ovOXm1REo
-ASPF3l4YAAjOHsXqcPtFHKGJAQCiuA71d6CQ+qNZLuka/KVB/etkkJvDzvaTtiQQ
-QG+gAwkQJ95A2bhFXBtRDgEAqymMavroD5c/4+M/EZ3/d8wxfA9E3Fb/1mt4c2Zr
-NnkBAKYOM+pz/pncFnV+kF7h7TQEEYuGw1JhJVT/duA4lwsLuDMEXZwXARYJKwYB
-BAHaRw8BAQdAa76TmWuKuiR1bnNV1FUE6oQ4C8A+UiQb8x0k1z2DmTKIfgQYFggA
-JgIbIBYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZwBQkLMdZgAAoJECfeQNm4
-RVwb8TkA/RkBu9Ev8iDE5nvn8YF8FRiY56Z5d+SBPG4VvrCzXrmlAP46wUjIRpkM
-rTbb1GMbvYnkeOrBs/qiWjEtHHc3ZLMWD7g4BF2cFygSCisGAQQBl1UBBQEBB0AO
-0t3BUxLuokTqKVcheFAZd4UKxAGznPQlvsVyhWWIEgMBCAeIfgQYFggAJgIbDBYh
-BLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZwBQkLMdY5AAoJECfeQNm4RVwbXscA
-/A8zRRTCwQKxJ8iz5jmTcVFAhl2vD781Dtv8NvcWd5t8APwIwcuFVZZA3yayhIxi
-3aqYpMRxpn2t6Nswax1MIM8DBQ==
-=dzEV
+mDMEaNLjzBYJKwYBBAHaRw8BAQdAoegj6iXzJgxBkW8LyRS8ANRzp0LqyFbW1kRr
+Z4VtVRK0HlRvbSBBbGV4YW5kZXIgPHdvcmtAZml6ei5idXp6PoiQBBMWCAA4FiEE
+0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7GoCGwEFCwkIBwIGFQoJCAsCBBYCAwEC
+HgECF4AACgkQQngpn7hPaHXNRAEAxOHPULwbf/FIzS7spmdSYrcCX/foaB78rpCT
+/MzDPvMBANy0PcseR1ZxoHZDcAsYDa0CSCrO6oLwPFriVss3RA0GtB1Ub20gQWxl
+eGFuZGVyIDx0b21AZml6ei5idXp6PoiTBBMWCAA7AhsBBQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7MkCGQEACgkQQngp
+n7hPaHX9fQEA4ngwEKr0nlKxH5bQV9u/EJeI3wbSgBjlnyTQuI79AB4BAO6+frGt
+8S+p6qFZ4ufqyGPfklxPeOJLSYk0PLKVNMcHuDMEaNLm8xYJKwYBBAHaRw8BAQdA
+HDhppS6yD8j1Bb/i6ku16uQ3qhshDNA9cOQeMxBae9aI9QQYFggAJhYhBNJyyNYW
+fyaFlGdmb0J4KZ+4T2h1BQJo0ubzAhsCBQkDwmcAAIEJEEJ4KZ+4T2h1diAEGRYI
+AB0WIQS9v3ap15pUELURqaY2yZ6LPDnYXwUCaNLm8wAKCRA2yZ6LPDnYXyaNAPsF
+gR37jEqfgEByVsoKY6bB82T79o9d4FQe1iPsURyuLwD/fkQyV3NwGjysxkoZqYmK
+mXJYqtWRBTe2G2UUkm6E/QafHwD+IbkCZ6sGTcexsqzex5x6U8TOvbdVS4dKjSf1
+nVRGxvwBAPiIJsXWVuwmskWMDpcaW/qgQ8hOEuq7/vlkZDGOnMgOuDgEaNLnDBIK
+KwYBBAGXVQEFAQEHQBcOCDGnrRwv51c5B7QVLMkLC2UKUzPPrahLZHT3RWhmAwEI
+B4h+BBgWCAAmFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS5wwCGwwFCQPCZwAA
+CgkQQngpn7hPaHUZIAD/ZwQ9sLIwuO5qPFAAkqcaNyt68O6WkD8sKaq1r/TPviAA
+/j92d7cRUIkJtS8odRYlK51r9eMeTGh2npaO+j3VKCgBuDMEaNLnJRYJKwYBBAHa
+Rw8BAQdAPT7jOLbozd5hacityJHniQ6UbHN+AJcb6jh5rXOnOuSIfgQYFggAJhYh
+BNJyyNYWfyaFlGdmb0J4KZ+4T2h1BQJo0uclAhsgBQkDwmcAAAoJEEJ4KZ+4T2h1
+rREA/3QE6suVUDl4OS2tCi4z2fh/7kjt29I3IFo+/B0AOumgAP0ao8FGqJyFC8YA
+7V6T4qrXHbhlqTeofGhQ+iu7HqZVCw==
+=OfDR
 -----END PGP PUBLIC KEY BLOCK-----

ansible/roles/build/files/gpg_work.asc

@@ -1,27 +1,27 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
-mDMEXZwWGhYJKwYBBAHaRw8BAQdAfv7qozKkmf4D+5PDzADsMm4aAKDGLha7+Cu0
-0H+RsWG0LVRob21hcyBBbGV4YW5kZXIgPFRob21hc0EuQWxleGFuZGVyQGhtaG4u
-b3JnPoiQBBMWCAA4FiEEuEgVk2PCh3kXlUvhJ95A2bhFXBsFAmULicsCGwMFCwkI
-BwIGFQoJCAsCBBYCAwECHgECF4AACgkQJ95A2bhFXBsUtQD9GWPdWc/nSmO0Gp7p
-DzxrieliriAnO+ZCHp31mFbMtToBAPxPYN9y4kgSiXhLiFLoRK5k5FCspksTSitg
-0CbXDE4LuDgEXZwWGhIKKwYBBAGXVQEFAQEHQK202EIAwTBuxARUygOvn+AloMJd
-ui39m+nMghn1MNo+AwEIB4h4BBgWCAAgFiEEuEgVk2PCh3kXlUvhJ95A2bhFXBsF
-Al2cFhoCGwwACgkQJ95A2bhFXBtNzAEAq5I6xPjIbb23xmhxh5cM/UJxdGedfWMy
-vF6/JtDvtPUBAPQRQn5AMwTOA+CSnliYf7ZjfVOlHscy60XWPlvXLoAJuDMEXZwW
-yhYJKwYBBAHaRw8BAQdAPyIL4EGg4T5JO9q2kpVDy2WjMiXz3nZXwYW4GLoTYkiI
-9QQYFggAJgIbAhYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZhBQkLMdaXAIF2
-IAQZFggAHRYhBIHmRDmWdVAusSUWutOhecmlPA7eBQJdnBbKAAoJENOhecmlPA7e
-jJ4A/iq7N2mMhx+ovOXm1REoASPF3l4YAAjOHsXqcPtFHKGJAQCiuA71d6CQ+qNZ
-Luka/KVB/etkkJvDzvaTtiQQQG+gAwkQJ95A2bhFXBtRDgEAqymMavroD5c/4+M/
-EZ3/d8wxfA9E3Fb/1mt4c2ZrNnkBAKYOM+pz/pncFnV+kF7h7TQEEYuGw1JhJVT/
-duA4lwsLuDMEXZwXARYJKwYBBAHaRw8BAQdAa76TmWuKuiR1bnNV1FUE6oQ4C8A+
-UiQb8x0k1z2DmTKIfgQYFggAJgIbIBYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJl
-C4ZwBQkLMdZgAAoJECfeQNm4RVwb8TkA/RkBu9Ev8iDE5nvn8YF8FRiY56Z5d+SB
-PG4VvrCzXrmlAP46wUjIRpkMrTbb1GMbvYnkeOrBs/qiWjEtHHc3ZLMWD7g4BF2c
-FygSCisGAQQBl1UBBQEBB0AO0t3BUxLuokTqKVcheFAZd4UKxAGznPQlvsVyhWWI
-EgMBCAeIfgQYFggAJgIbDBYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZwBQkL
-MdY5AAoJECfeQNm4RVwbXscA/A8zRRTCwQKxJ8iz5jmTcVFAhl2vD781Dtv8NvcW
-d5t8APwIwcuFVZZA3yayhIxi3aqYpMRxpn2t6Nswax1MIM8DBQ==
-=0HtE
+mDMEaNLjzBYJKwYBBAHaRw8BAQdAoegj6iXzJgxBkW8LyRS8ANRzp0LqyFbW1kRr
+Z4VtVRK0HlRvbSBBbGV4YW5kZXIgPHdvcmtAZml6ei5idXp6PoiQBBMWCAA4FiEE
+0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7GoCGwEFCwkIBwIGFQoJCAsCBBYCAwEC
+HgECF4AACgkQQngpn7hPaHXNRAEAxOHPULwbf/FIzS7spmdSYrcCX/foaB78rpCT
+/MzDPvMBANy0PcseR1ZxoHZDcAsYDa0CSCrO6oLwPFriVss3RA0GtB1Ub20gQWxl
+eGFuZGVyIDx0b21AZml6ei5idXp6PoiTBBMWCAA7AhsBBQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7MkCGQEACgkQQngp
+n7hPaHX9fQEA4ngwEKr0nlKxH5bQV9u/EJeI3wbSgBjlnyTQuI79AB4BAO6+frGt
+8S+p6qFZ4ufqyGPfklxPeOJLSYk0PLKVNMcHuDMEaNLm8xYJKwYBBAHaRw8BAQdA
+HDhppS6yD8j1Bb/i6ku16uQ3qhshDNA9cOQeMxBae9aI9QQYFggAJhYhBNJyyNYW
+fyaFlGdmb0J4KZ+4T2h1BQJo0ubzAhsCBQkDwmcAAIEJEEJ4KZ+4T2h1diAEGRYI
+AB0WIQS9v3ap15pUELURqaY2yZ6LPDnYXwUCaNLm8wAKCRA2yZ6LPDnYXyaNAPsF
+gR37jEqfgEByVsoKY6bB82T79o9d4FQe1iPsURyuLwD/fkQyV3NwGjysxkoZqYmK
+mXJYqtWRBTe2G2UUkm6E/QafHwD+IbkCZ6sGTcexsqzex5x6U8TOvbdVS4dKjSf1
+nVRGxvwBAPiIJsXWVuwmskWMDpcaW/qgQ8hOEuq7/vlkZDGOnMgOuDgEaNLnDBIK
+KwYBBAGXVQEFAQEHQBcOCDGnrRwv51c5B7QVLMkLC2UKUzPPrahLZHT3RWhmAwEI
+B4h+BBgWCAAmFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS5wwCGwwFCQPCZwAA
+CgkQQngpn7hPaHUZIAD/ZwQ9sLIwuO5qPFAAkqcaNyt68O6WkD8sKaq1r/TPviAA
+/j92d7cRUIkJtS8odRYlK51r9eMeTGh2npaO+j3VKCgBuDMEaNLnJRYJKwYBBAHa
+Rw8BAQdAPT7jOLbozd5hacityJHniQ6UbHN+AJcb6jh5rXOnOuSIfgQYFggAJhYh
+BNJyyNYWfyaFlGdmb0J4KZ+4T2h1BQJo0uclAhsgBQkDwmcAAAoJEEJ4KZ+4T2h1
+rREA/3QE6suVUDl4OS2tCi4z2fh/7kjt29I3IFo+/B0AOumgAP0ao8FGqJyFC8YA
+7V6T4qrXHbhlqTeofGhQ+iu7HqZVCw==
+=OfDR
 -----END PGP PUBLIC KEY BLOCK-----

ansible/roles/build/tasks/linux.yaml

@@ -40,11 +40,11 @@
   command: pacman-key -a -
   args:
     stdin: "{{ lookup('file', pgp_key|default('gpg.asc')) }}"
-  when: '"B848159363C2877917954BE127DE40D9B8455C1B" not in pacmankeys.stdout'
+  when: '"D272C8D6167F26859467666F4278299FB84F6875" not in pacmankeys.stdout'
   register: my_key_imported
 
 - name: Sign my signing key
-  command: pacman-key --lsign-key "B848159363C2877917954BE127DE40D9B8455C1B"
+  command: pacman-key --lsign-key "D272C8D6167F26859467666F4278299FB84F6875"
   when: my_key_imported.changed
 
 - name: Build the aurutils package
@@ -103,7 +103,8 @@
     - /var/cache/pacman/custom/
 
 - name: Create custom repo db
-  command: repo-add --new --sign /var/cache/pacman/custom/custom.db.tar "/home/{{ build_user.name }}/.config/ansible_deploy/aurutils/aurutils-*-any.pkg.tar.*"
+  # shell: repo-add --new --sign /var/cache/pacman/custom/custom.db.tar "/home/{{ build_user.name }}/.config/ansible_deploy/aurutils/aurutils-*-any.pkg.tar.*"
+  command: repo-add --new --sign /var/cache/pacman/custom/custom.db.tar
   become: true
   become_user: "{{ build_user.name }}"
   args:

ansible/roles/chromium/files/chromium-flags.conf

@@ -0,0 +1,2 @@
+--ozone-platform-hint=auto
+--enable-features=VaapiVideoDecoder,VaapiIgnoreDriverChecks,Vulkan,DefaultANGLEVulkan,VulkanFromANGLE,AcceleratedVideoEncoder

ansible/roles/chromium/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - users

ansible/roles/chromium/tasks/common.yaml

@@ -0,0 +1,55 @@
+# - name: Create directories
+#   file:
+#     name: "{{ item }}"
+#     state: directory
+#     mode: 0755
+#     owner: root
+#     group: wheel
+#   loop:
+#     - /foo/bar
+
+# - name: Install scripts
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ item.dest }}"
+#     mode: 0755
+#     owner: root
+#     group: wheel
+#   loop:
+#     - src: foo.bash
+#       dest: /usr/local/bin/foo
+
+# - name: Install Configuration
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ item.dest }}"
+#     mode: 0600
+#     owner: root
+#     group: wheel
+#   loop:
+#     - src: foo.conf
+#       dest: /usr/local/etc/foo.conf
+
+# - name: Clone Source
+#   git:
+#     repo: "https://foo.bar/baz.git"
+#     dest: /foo/bar
+#     version: "v1.0.2"
+#     force: true
+#   diff: false
+
+- import_tasks: tasks/freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/linux.yaml
+  when: 'os_flavor == "linux"'
+
+- include_tasks:
+    file: tasks/peruser.yaml
+    apply:
+      become: yes
+      become_user: "{{ initialize_user }}"
+  when: users is defined
+  loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}"
+  loop_control:
+    loop_var: initialize_user

ansible/roles/chromium/tasks/freebsd.yaml

@@ -0,0 +1,5 @@
+# - name: Install packages
+#   package:
+#     name:
+#       - foo
+#     state: present

ansible/roles/chromium/tasks/linux.yaml

@@ -0,0 +1,7 @@
+# Check chrome://gpu/ to confirm hardware video decoding and vulkan rendering is working.
+
+- name: Install packages
+  package:
+    name:
+      - chromium
+    state: present

ansible/roles/chromium/tasks/main.yaml

@@ -0,0 +1,2 @@
+- import_tasks: tasks/common.yaml
+  when: install_graphics

ansible/roles/chromium/tasks/peruser.yaml

@@ -0,0 +1,29 @@
+- include_role:
+    name: per_user
+
+# - name: Create directories
+#   file:
+#     name: "{{ account_homedir.stdout }}/{{ item }}"
+#     state: directory
+#     mode: 0700
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - ".config/foo"
+
+# - name: Copy files
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ account_homedir.stdout }}/{{ item.dest }}"
+#     mode: 0600
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - src: foo.conf
+#       dest: .config/foo/foo.conf
+
+- import_tasks: tasks/peruser_freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/peruser_linux.yaml
+  when: 'os_flavor == "linux"'

ansible/roles/chromium/tasks/peruser_linux.yaml

@@ -0,0 +1,10 @@
+- name: Copy files
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ account_homedir.stdout }}/{{ item.dest }}"
+    mode: 0600
+    owner: "{{ account_name.stdout }}"
+    group: "{{ group_name.stdout }}"
+  loop:
+    - src: chromium-flags.conf
+      dest: .config/chromium-flags.conf

ansible/roles/devfs/files/homeserver_devfs.rules

@@ -17,3 +17,9 @@ add include $devfsrules_hide_all
 add include $devfsrules_unhide_basic
 add include $devfsrules_unhide_login
 add path 'bpf*' unhide
+
+[tajailrand=15]
+add include $devfsrules_hide_all
+add include $devfsrules_unhide_basic
+add include $devfsrules_unhide_login
+add path urandom unhide

ansible/roles/docker/tasks/linux.yaml

@@ -2,6 +2,8 @@
   package:
     name:
       - docker
+      - docker-compose
+      - docker-buildx
     state: present
 
 - name: Create docker zfs dataset

ansible/roles/dummynet/files/dnctl.conf

@@ -0,0 +1,2 @@
+pipe 1 config bw 100KByte/s
+pipe 2 config

ansible/roles/dummynet/files/dummynet

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+#
+
+# PROVIDE: dummynet
+# BEFORE: pf ipfw
+# KEYWORD: nojailvnet
+
+. /etc/rc.subr
+
+name="dummynet"
+desc="Dummynet packet queuing and scheduling"
+rcvar="${name}_enable"
+load_rc_config $name
+start_cmd="${name}_start"
+required_files="$dummynet_rules"
+required_modules="dummynet"
+
+dummynet_start()
+{
+	startmsg -n "Enabling ${name}"
+        cat "$dnctl_rules" | while read l; do
+          dnctl $l
+        done
+	startmsg '.'
+}
+
+run_rc_command $*

ansible/roles/dummynet/files/dummynet_rc.conf

@@ -0,0 +1,2 @@
+dummynet_enable="YES"
+dummynet_rules="/etc/dnctl.conf"

ansible/roles/dummynet/tasks/common.yaml

@@ -0,0 +1,55 @@
+# - name: Create directories
+#   file:
+#     name: "{{ item }}"
+#     state: directory
+#     mode: 0755
+#     owner: root
+#     group: wheel
+#   loop:
+#     - /foo/bar
+
+# - name: Install scripts
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ item.dest }}"
+#     mode: 0755
+#     owner: root
+#     group: wheel
+#   loop:
+#     - src: foo.bash
+#       dest: /usr/local/bin/foo
+
+# - name: Install Configuration
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ item.dest }}"
+#     mode: 0600
+#     owner: root
+#     group: wheel
+#   loop:
+#     - src: foo.conf
+#       dest: /usr/local/etc/foo.conf
+
+# - name: Clone Source
+#   git:
+#     repo: "https://foo.bar/baz.git"
+#     dest: /foo/bar
+#     version: "v1.0.2"
+#     force: true
+#   diff: false
+
+- import_tasks: tasks/freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/linux.yaml
+  when: 'os_flavor == "linux"'
+
+- include_tasks:
+    file: tasks/peruser.yaml
+    apply:
+      become: yes
+      become_user: "{{ initialize_user }}"
+  when: users is defined
+  loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}"
+  loop_control:
+    loop_var: initialize_user

ansible/roles/dummynet/tasks/freebsd.yaml

@@ -0,0 +1,30 @@
+- name: Install Configuration
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0600
+    owner: root
+    group: wheel
+  loop:
+    - src: "{{ dummynet_config }}"
+      dest: /etc/dnctl.conf
+
+- name: Install rc script
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}"
+    owner: root
+    group: wheel
+    mode: 0755
+  loop:
+    - src: dummynet
+
+- name: Install service configuration
+  copy:
+    src: "files/{{ item }}_rc.conf"
+    dest: "/etc/rc.conf.d/{{ item }}"
+    mode: 0644
+    owner: root
+    group: wheel
+  loop:
+    - dummynet

ansible/roles/dummynet/tasks/linux.yaml

@@ -0,0 +1,29 @@
+# - name: Build aur packages
+#   register: buildaur
+#   become_user: "{{ build_user.name }}"
+#   command: "aurutils-sync --no-view {{ item }}"
+#   args:
+#     creates: "/var/cache/pacman/custom/{{ item }}-*.pkg.tar.*"
+#   loop:
+#     - foo
+
+# - name: Update cache
+#   when: buildaur.changed
+#   pacman:
+#     name: []
+#     state: present
+#     update_cache: true
+
+# - name: Install packages
+#   package:
+#     name:
+#       - foo
+#     state: present
+
+# - name: Enable services
+#   systemd:
+#     enabled: yes
+#     name: "{{ item }}"
+#     daemon_reload: yes
+#   loop:
+#     - foo.service

ansible/roles/dummynet/tasks/main.yaml

@@ -0,0 +1,2 @@
+- import_tasks: tasks/common.yaml
+  when: (dummynet_config is defined and os_flavor == "freebsd") or (os_flavor == "linux")

ansible/roles/dummynet/tasks/peruser.yaml

@@ -0,0 +1,29 @@
+- include_role:
+    name: per_user
+
+# - name: Create directories
+#   file:
+#     name: "{{ account_homedir.stdout }}/{{ item }}"
+#     state: directory
+#     mode: 0700
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - ".config/foo"
+
+# - name: Copy files
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ account_homedir.stdout }}/{{ item.dest }}"
+#     mode: 0600
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - src: foo.conf
+#       dest: .config/foo/foo.conf
+
+- import_tasks: tasks/peruser_freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/peruser_linux.yaml
+  when: 'os_flavor == "linux"'

ansible/roles/emacs/files/early-init.el

@@ -1,7 +1,7 @@
-(setq gc-cons-threshold (* 128 1024 1024)) ;; Increase garbage collection threshold for performance (default 800000)
+(setq gc-cons-threshold (* 128 1024 1024)) ;; 128MiB Increase garbage collection threshold for performance (default 800000)
 ;; Increase amount of data read from processes, default 4k
 (when (version<= "27.0" emacs-version)
-  (setq read-process-output-max (* 1024 1024)) ;; 1mb
+  (setq read-process-output-max (* 10 1024 1024)) ;; 10MiB
   )
 
 ;; Suppress warnings

ansible/roles/emacs/files/elisp/base-extensions.el

@@ -51,17 +51,27 @@
 ;; Persist history over Emacs restarts. Vertico sorts by history position.
 (use-package savehist
   ;; This is an emacs built-in but we're pulling the latest version
+  :pin gnu
   :config
   (savehist-mode))
 
 (use-package which-key
+  :pin gnu
   :diminish
   :config
   (which-key-mode))
 
 (use-package windmove
-  :config
-  (windmove-default-keybindings))
+  ;; This is an emacs built-in but we're pulling the latest version
+  :pin gnu
+  :bind
+  (
+   ("S-<up>" . windmove-up)
+   ("S-<right>" . windmove-right)
+   ("S-<down>" . windmove-down)
+   ("S-<left>" . windmove-left)
+   )
+  )
 
 (setq tramp-default-method "ssh")
 

ansible/roles/emacs/files/elisp/base.el

@@ -63,6 +63,9 @@
  show-trailing-whitespace t
  ;; Remove the line when killing it with ctrl-k
  kill-whole-line t
+
+ ;; Show the current project in the mode line
+ project-mode-line t
  )
 
 ;; (setq-default fringes-outside-margins t)

ansible/roles/emacs/files/elisp/lang-cmake.el

@@ -0,0 +1,18 @@
+(require 'common-lsp)
+
+(use-package cmake-mode
+  :commands cmake-mode
+  :hook (
+         (cmake-mode . (lambda ()
+                         (eglot-ensure)
+                         (defclass my/eglot-cmake (eglot-lsp-server) ()
+                           :documentation
+                           "Own eglot server class.")
+
+                         (add-to-list 'eglot-server-programs
+                                      '(cmake-mode . (my/eglot-cmake "cmake-language-server")))
+                         ))
+         )
+  )
+
+(provide 'lang-cmake)

ansible/roles/emacs/files/elisp/lang-d2.el

@@ -0,0 +1,16 @@
+(defun d2-format-buffer ()
+  "Run prettier."
+  (interactive)
+  (run-command-on-buffer "d2" "fmt" "-")
+  )
+
+(use-package d2-mode
+  :commands (d2-mode)
+  :hook (
+         (d2-mode . (lambda ()
+                      ;; (add-hook 'before-save-hook 'd2-format-buffer nil 'local)
+                      ))
+         )
+  )
+
+(provide 'lang-d2)

ansible/roles/emacs/files/elisp/lang-nix.el

@@ -0,0 +1,22 @@
+(require 'common-lsp)
+(require 'util-tree-sitter)
+
+(use-package nix-mode
+  :mode (("\\.nix\\'" . nix-mode)
+         )
+  :commands nix-mode
+  :hook (
+         (nix-mode . (lambda ()
+                       (eglot-ensure)
+                       (defclass my/eglot-nix (eglot-lsp-server) ()
+                         :documentation
+                         "Own eglot server class.")
+
+                       (add-to-list 'eglot-server-programs
+                                    '(nix-mode . (my/eglot-nix "nixd")))
+                       (add-hook 'before-save-hook 'eglot-format-buffer nil 'local)
+                       ))
+         )
+  )
+
+(provide 'lang-nix)

ansible/roles/emacs/files/elisp/lang-org.el

@@ -1,14 +1,23 @@
 (use-package org
   :ensure nil
   :commands org-mode
-  :bind (
+  :bind (:map org-mode-map
          ("C-c l" . org-store-link)
          ("C-c a" . org-agenda)
+         ("S-<up>" . org-shiftup)
+         ("S-<right>" . org-shiftright)
+         ("S-<down>" . org-shiftdown)
+         ("S-<left>" . org-shiftleft)
          )
   :hook (
          (org-mode . (lambda ()
                        (org-indent-mode +1)
-                        ))
+                       ))
+         ;; Make windmove work in Org mode:
+         (org-shiftup-final . windmove-up)
+         (org-shiftleft-final . windmove-left)
+         (org-shiftdown-final . windmove-down)
+         (org-shiftright-final . windmove-right)
          )
   :config
   (require 'org-tempo)
@@ -36,6 +45,8 @@
 
   ;; TODO: There is an option to set the compiler, could be better than manually doing this here https://orgmode.org/manual/LaTeX_002fPDF-export-commands.html
   ;; (setq org-latex-compiler "lualatex")
+  ;; TODO: nixos latex page recommends this line, figure out what it does / why its needed:
+  ;; (setq org-preview-latex-default-process 'dvisvgm)
   (setq org-latex-pdf-process
         '("lualatex -shell-escape -interaction nonstopmode -output-directory %o %f"
           "lualatex -shell-escape -interaction nonstopmode -output-directory %o %f"
@@ -76,4 +87,8 @@
 (use-package gnuplot)
 (use-package graphviz-dot-mode)
 
+(use-package htmlize
+  ;; For syntax highlighting when exporting to HTML.
+  )
+
 (provide 'lang-org)

ansible/roles/emacs/files/elisp/util-tree-sitter.el

@@ -4,6 +4,8 @@
   :commands (treesit-install-language-grammar treesit-ready-p)
   :init
   (setq treesit-language-source-alist '())
+  :custom
+  (treesit-max-buffer-size 209715200) ;; 200MiB
   :config
   ;; Default to the max level of detail in treesitter highlighting. This
   ;; can be overridden in each language's use-package call with:

ansible/roles/emacs/files/init.el

@@ -36,4 +36,10 @@
 
 (require 'lang-xml)
 
+(require 'lang-nix)
+
+(require 'lang-cmake)
+
+(require 'lang-d2)
+
 (load-directory autoload-directory)

ansible/roles/emacs/files/plainmacs

@@ -15,7 +15,8 @@ INIT_SCRIPT=$(cat <<EOF
   ;; Set default font
   (set-face-attribute 'default nil :height 100 :width 'regular :weight 'regular :family "Cascadia Mono")
   ;; Set fallback font for unicode glyphs
-  (set-fontset-font "fontset-default" nil (font-spec :name "Noto Color Emoji"))
+  (when (display-graphic-p)
+    (set-fontset-font "fontset-default" nil (font-spec :name "Noto Color Emoji")))
   (menu-bar-mode -1)
   (when (fboundp 'tool-bar-mode)
     (tool-bar-mode -1))

ansible/roles/emacs/files/plainmacs_init.el

@@ -11,7 +11,8 @@
   ;; Set default font
   (set-face-attribute 'default nil :height 100 :width 'regular :weight 'regular :family "Cascadia Mono")
   ;; Set fallback font for unicode glyphs
-  (set-fontset-font "fontset-default" nil (font-spec :name "Noto Color Emoji"))
+  (when (display-graphic-p)
+    (set-fontset-font "fontset-default" nil (font-spec :name "Noto Color Emoji")))
   (menu-bar-mode -1)
   (when (fboundp 'tool-bar-mode)
     (tool-bar-mode -1))

ansible/roles/emacs/meta/main.yaml

@@ -7,3 +7,5 @@ dependencies:
     when: 'emacs_flavor == "full"'
   - role: terraform
     when: 'emacs_flavor == "full"'
+  - role: nix
+    when: 'emacs_flavor == "full"'

ansible/roles/emacs/tasks/linux.yaml

@@ -15,6 +15,7 @@
       - typescript-language-server
       - shellcheck
       - vscode-css-languageserver
+      - d2 # Generating diagrams
     state: present
 
 - name: Create directories

ansible/roles/firefox/defaults/main.yaml

@@ -7,9 +7,38 @@ firefox_config:
   dom.security.https_only_mode_ever_enabled: true
   extensions.activeThemeID: "firefox-compact-dark@mozilla.org"
   # Disable ads
-  extensions.pocket.enabled: false
   browser.newtabpage.activity-stream.showSponsored: false
   browser.newtabpage.activity-stream.showSponsoredTopSites: false
   browser.newtabpage.activity-stream.feeds.section.topstories: false
   browser.newtabpage.pinned: "[]"
   browser.newtabpage.activity-stream.section.highlights.includePocket: false
+  # Disable cache when devtools are open.
+  devtools.cache.disabled: true
+  # Do not track header.
+  privacy.donottrackheader.enabled: true
+  # Tell websites not to share or sell my data.
+  privacy.globalprivacycontrol.enabled: true
+  # Disable "studies" (slice testing)
+  app.shield.optoutstudies.enabled: false
+  # Disable battery status, used to track users.
+  dom.battery.enabled: false
+  # Disable that websites can get notifications if you copy, paste, or cut something from a web page, and it lets them know which part of the page had been selected.
+  #
+  # This breaks copying from BigQuery https://github.com/microsoft/monaco-editor/issues/1540
+  # dom.event.clipboardevents.enabled: false
+  # Isolates all browser identifier sources (e.g. cookies) to the first party domain, with the goal of preventing tracking across different domains.
+  privacy.firstparty.isolate: true
+  # Do not preload URLs that auto-complete in the address bar.
+  browser.urlbar.speculativeConnect.enabled: false
+  # Do not resist fingerprinting because that tells websites to use light mode.
+  # https://bugzilla.mozilla.org/show_bug.cgi?id=1732114
+  privacy.resistFingerprinting: null # (default false)
+  # Instead, enable fingerprinting protection, which allows configuring an override.
+  privacy.fingerprintingProtection: true
+  # Allow sending dark mode preference to websites.
+  # Allow sending timezone to websites.
+  privacy.fingerprintingProtection.overrides: "+AllTargets,-CSSPrefersColorScheme,-JSDateTimeUTC,-CanvasExtractionBeforeUserInputIsBlocked,-CanvasImageExtractionPrompt,-CanvasExtractionFromThirdPartiesIsBlocked"
+  # Disable weather on new tab page
+  browser.newtabpage.activity-stream.showWeather: false
+  browser.ml.chat.enabled: false
+  browser.ml.enabled: false

ansible/roles/firefox/tasks/linux.yaml

@@ -3,4 +3,5 @@
     name:
       - libfido2
       - firefox-developer-edition
+      - speech-dispatcher # For TTS
     state: present

ansible/roles/firefox/tasks/peruser.yaml

@@ -10,12 +10,21 @@
   register: firefox_about_config
 
 - name: Configure Firefox about:config
+  when: item[1].value != None
   lineinfile:
     path: "{{ item[0].path }}"
     regexp: '"{{ item[1].key }}", [^")\n]*\)'
     line: 'user_pref("{{ item[1].key }}", {{ item[1].value | to_json }});'
   loop: "{{ firefox_about_config.files | product(firefox_config | dict2items) | list }}"
 
+- name: Configure Firefox about:config
+  when: item[1].value == None
+  lineinfile:
+    path: "{{ item[0].path }}"
+    regexp: '"{{ item[1].key }}", [^")\n]*\)'
+    state: absent
+  loop: "{{ firefox_about_config.files | product(firefox_config | dict2items) | list }}"
+
 - import_tasks: tasks/peruser_freebsd.yaml
   when: 'os_flavor == "freebsd"'
 

ansible/roles/firewall/files/homeserver_pf.conf

@@ -1,9 +1,10 @@
-ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
-not_ext_if = "{ !igb0 !igb1 !ix0 !ix1 !wlan0 }"
+ext_if = "{ igb0 igb1 ix0 ix1 linfi_host }"
+not_ext_if = "{ !igb0 !igb1 !ix0 !ix1 !linfi_host }"
 jail_nat_v4 = "{ 10.215.1.0/24 }"
 not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
 restricted_nat_v4 = "{ 10.215.2.0/24 }"
 not_restricted_nat_v4 = "{ any, !10.215.2.0/24 }"
+rfc1918 = "{ 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ wgh wgf }"
@@ -18,39 +19,50 @@ unifi_ports = "{ 8443 3478 10001 8080 1900 8843 8880 6789 5514 }"
 set skip on lo
 
 # queueing
-# altq on wlan0 cbq queue { def, stuff }
+# altq on linfi_host cbq queue { def, stuff }
 # queue def cbq(default borrow)
 # queue stuff bandwidth	8Mb cbq { dagger }
 # queue dagger cbq(borrow)
 
 # redirections
-nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (linfi_host)
 rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 172.16.0.1 port 53
 
 # cloak
-nat pass on $ext_if inet from 10.215.2.0/24 to !10.215.2.0/24 -> (wlan0)
+nat pass on $ext_if inet from 10.215.2.0/24 to !10.215.2.0/24 -> (linfi_host)
 rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.2.1 port 53 -> 172.16.0.1 port 53
 
 # bastion
-rdr pass on $ext_if inet proto tcp from { any, !10.215.1.0/24, !10.215.2.0/24 } to any port 8081 -> 10.215.1.217 port 443
+rdr pass on $ext_if inet proto {tcp, udp} from { any, !10.215.1.0/24, !10.215.2.0/24 } to any port 8081 -> 10.215.1.217 port 443
 nat pass on jail_nat proto {tcp, udp} from any to 10.215.1.217 port 443 -> 10.215.1.1
 nat pass on restricted_nat proto {tcp, udp} from 10.215.1.217/32 to 10.215.2.2 port 8081 -> 10.215.2.1
 
 
 # cloak -> olddagger
-rdr pass on $ext_if inet proto tcp from $not_restricted_nat_v4 to any port 8082 -> 10.215.2.2 port 8082
+rdr pass on $ext_if inet proto {tcp, udp} from $not_restricted_nat_v4 to any port 8082 -> 10.215.2.2 port 8082
 nat pass on restricted_nat proto {tcp, udp} from any to 10.215.2.2 port 8082 -> 10.215.2.1
 
+# cloak -> dagger old
+rdr pass on $ext_if inet proto {tcp, udp} from $not_restricted_nat_v4 to any port 8083 -> 10.215.2.2 port 8083
+nat pass on restricted_nat proto {tcp, udp} from any to 10.215.2.2 port 8083 -> 10.215.2.1
+
 # -> sftp
 # TODO: Limit bandwidth for sftp
-rdr pass on $ext_if inet proto tcp from $not_jail_nat_v4 to any port 8022 -> 10.215.1.216 port 22
+rdr pass on $ext_if inet proto {tcp, udp} from $not_jail_nat_v4 to any port 8022 -> 10.215.1.216 port 22
 nat pass on jail_nat proto {tcp, udp} from any to 10.215.1.216 port 22 -> 10.215.1.1
 
 # Forward ports for unifi controller
-# rdr pass on $ext_if inet proto tcp from any to any port 65022 -> 10.213.177.8 port 22
+# rdr pass on $ext_if inet proto {tcp, udp} from any to any port 65022 -> 10.213.177.8 port 22
 rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports -> 10.215.1.202
 
+# -> momlaptop
+rdr pass on $ext_if inet proto {tcp, udp} from $not_jail_nat_v4 to any port 8033 -> 10.215.1.218 port 443
+nat pass on jail_nat proto {tcp, udp} from any to 10.215.1.218 port 443 -> 10.215.1.1
+
 # filtering
+# match in on jail_nat from any to any dnpipe(1, 2)
+# match in on restricted_nat from any to any dnpipe(1, 2)
+
 block log all
 pass out on $ext_if
 

ansible/roles/firewall/files/mrmanager_pf.conf

@@ -2,7 +2,8 @@ ext_if = "lagg0"
 not_ext_if = "{ !lagg0 }"
 jail_nat_v4 = "{ 10.215.1.0/24 }"
 not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
-pub_k8s = "{ 74.80.180.136/29, !74.80.180.138 }"
+# pub_k8s = "{ 74.80.180.136/29, !74.80.180.138 }"
+pub_k8s = "{ 74.80.180.137, 74.80.180.139, 74.80.180.140, 74.80.180.141, 74.80.180.142 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ colo }"
@@ -33,20 +34,24 @@ scrub in on $ext_if all fragment reassemble
 
 # redirections
 nat on $ext_if inet from ! ($ext_if) to ! ($ext_if) -> ($ext_if)
-rdr pass proto {tcp, udp} from any to 10.215.1.1 port 53 tag REDIREXTERNAL -> 1.1.1.1 port 53
+rdr pass on jail_nat proto {tcp, udp} from any to 10.215.1.1 port 53 tag REDIREXTERNAL -> 1.1.1.1 port 53
 
-rdr pass on $ext_if proto {tcp, udp} to ($ext_if) port 6443 -> 10.215.1.204 port 6443
+rdr pass on $ext_if proto {tcp, udp} to ($ext_if) port 6443 tag REDIRINTERNAL -> 10.215.1.204 port 6443
 rdr pass on jail_nat proto {tcp, udp} to ($ext_if) port 6443 tag REDIRINTERNAL -> 10.215.1.204 port 6443
 
-rdr pass on $ext_if proto {tcp, udp} to ($ext_if) port 19993 -> 10.215.1.204 port 19993
+rdr pass on $ext_if proto {tcp, udp} to ($ext_if) port 19993 tag REDIRINTERNAL -> 10.215.1.204 port 19993
 rdr pass on jail_nat proto {tcp, udp} to ($ext_if) port 19993 tag REDIRINTERNAL -> 10.215.1.204 port 19993
 
-rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 65099 -> 10.215.1.210 port 22
+rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 65099 tag REDIRINTERNAL -> 10.215.1.210 port 22
 rdr pass proto {tcp, udp} from $jail_nat_v4 to ($ext_if) port 65099 tag REDIRINTERNAL -> 10.215.1.210 port 22
 
-rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 53 -> 10.215.1.211 port 53
+# log (to pflog1)
+rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 53 tag REDIRINTERNAL -> 10.215.1.211 port 53
 rdr pass proto {tcp, udp} from $jail_nat_v4 to ($ext_if) port 53 tag REDIRINTERNAL -> 10.215.1.211 port 53
 
+rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 65122 tag REDIRINTERNAL -> 10.215.1.219 port 22
+rdr pass proto {tcp, udp} from $jail_nat_v4 to ($ext_if) port 65122 tag REDIRINTERNAL -> 10.215.1.219 port 22
+
 nat pass tagged REDIRINTERNAL -> (jail_nat)
 nat pass tagged REDIREXTERNAL -> ($ext_if)
 
@@ -63,6 +68,9 @@ pass quick on $allow
 
 # Single interface kubernetes cluster is working with the following run on mrmanager:
 #   doas route add -host 74.80.180.139 -interface jail_nat
+#   doas route add -net 10.129.0.0/16 -interface jail_nat
+#   ? doas route -6 add -net '2620:11f:7001:7:ffff:ffff:0ad7:0100/120' -interface jail_nat
+#   ? doas ifconfig jail_nat inet6 2620:11f:7001:7:ffff:ffff:0ad7:0101/120
 #   doas sysctl net.link.ether.inet.proxyall=1
 # Plus this in pf.conf:
 #   pass quick from any to 74.80.180.139
@@ -72,6 +80,10 @@ pass in on jail_nat
 # Allow traffic from my machine to the jails/virtual machines
 pass out on jail_nat from (jail_nat:network)
 
+#pass quick in on $ext_if proto {tcp6, udp6} from any to 2620:11f:7001:7:ffff:ffff:0ad7:0100/120
+pass in quick on $ext_if from any to 2620:11f:7001:7:ffff:ffff:0ad7:0100/120
+pass out quick on jail_nat to 2620:11f:7001:7:ffff:ffff:0ad7:0100/120
+
 
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in

ansible/roles/firewall/files/odofreebsd_pf.conf

@@ -1,11 +1,11 @@
-ext_if = "{ wlan0 }"
-not_ext_if = "{ !wlan0 }"
+ext_if = "{ linfi_host }"
+not_ext_if = "{ !linfi_host }"
 jail_nat_v4 = "{ 10.215.1.0/24 }"
 not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
-dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }"
+rfc1918 = "{ 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }"
 
 dhcp = "{ bootpc, bootps }"
-#allow = "{ wgf wgh drmario colo }"
+allow = "{ wgf wgh drmario colo }"
 
 tcp_pass_in = "{ 22 }"
 udp_pass_in = "{ 53 51820 }"
@@ -16,8 +16,8 @@ udp_pass_in = "{ 53 51820 }"
 set skip on lo
 
 # redirections
-#nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
-#rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (linfi_host)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 172.16.0.1 port 53
 
 # Redirect jaeger ports to virtual machine.
 # nat pass on lo inet from 127.0.0.0/24 to 127.0.0.0/24 port {6831 6832 16686 14268} -> (jail_nat)
@@ -27,16 +27,18 @@ set skip on lo
 block log all
 pass out on $ext_if
 
-#pass in on jail_nat
+pass in on jail_nat
+# match in on jail_nat from any to any dnpipe 1
+# match in on jail_nat from any to $rfc1918 dnpipe 2
 # Allow traffic from my machine to the jails/virtual machines
-#pass out on jail_nat from $jail_nat_v4
+pass out on jail_nat from $jail_nat_v4
 
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
 # `service pf reload` but interfaces that we `skip` will not update (I
 # forget if its from adding, removing, or both. TODO: test to figure
 # it out). Also skipped interfaces are not subject to nat/rdr rules.
-#pass quick on $allow
+pass quick on $allow
 
 pass on $ext_if proto icmp all
 pass on $ext_if proto icmp6 all

ansible/roles/firewall/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - dummynet

ansible/roles/framework_laptop/files/disable_wifi_powersave_loader.conf

@@ -0,0 +1,3 @@
+  # Disable power save for wifi card because power save caused video stuttering in google meet on Linux. Both of these are currently the default on FreeBSD but I'm saving it just in case that default changes.
+compat.linuxkpi.iwlwifi_power_save="0"
+compat.linuxkpi.iwlwifi_mvm_power_scheme="1"

ansible/roles/framework_laptop/files/iwlwifi_modprobe.conf

@@ -1,5 +1,13 @@
-options iwlwifi power_save=1
+# Manually disable power save:
+# iw wlan0 set power_save off
 
-options iwlwifi uapsd_disable=0
+## High power:
+options iwlwifi power_save=0
+# options iwlwifi uapsd_disable=1
+options iwlmvm power_scheme=1 # 1-active, 2-balanced, 3-low power, default: 2 (int)
 
-options iwlmvm power_scheme=3
+## Low power:
+# options iwlwifi power_save=1
+# ? power_level:default power save level (range from 1 - 5, default: 1) (int)
+# options iwlwifi uapsd_disable=0
+# options iwlmvm power_scheme=3

ansible/roles/framework_laptop/files/launch_windows.bash

@@ -0,0 +1,285 @@
+#!/usr/local/bin/bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Share a host directory to the guest via 9pfs.
+#
+# Inside the VM run:
+#   mount -t virtfs -o trans=virtio sharename /some/vm/path
+#   mount -t 9p -o cache=mmap -o msize=512000 sharename /mnt/9p
+#   mount -t 9p -o trans=virtio,cache=mmap,msize=512000 sharename /path/to/mountpoint
+# bhyve_options="-s 28,virtio-9p,sharename=/"
+
+# Enable Sound
+# bhyve_options="-s 16,hda,play=/dev/dsp,rec=/dev/dsp"
+
+# Example usage:
+#
+#   doas bhyve_netgraph_bridge create-disk zdata/vm/poudriere /vm/poudriere 10
+#   doas bhyve_netgraph_bridge start poudriere zdata/vm/poudriere /vm/poudriere /vm/iso/FreeBSD-13.2-RELEASE-amd64-bootonly.iso
+#   doas bhyve_netgraph_bridge start poudriere zdata/vm/poudriere /vm/poudriere
+
+: ${VERBOSE:="NO"} # or YES
+: ${CPU_CORES:="1"}
+: ${MEMORY:="1G"}
+: ${NETWORK:="NAT"} # or RAW or BOTH
+: ${IP_RANGE:="10.215.1.1/24"} # Ignored for RAW networks
+: ${INTERFACE_NAME:="jail_nat"} # or the external interface like lagg0 for RAW networks
+: ${BRIDGE_NAME:="bridge_$INTERFACE_NAME"} # or bridge_raw for RAW networks
+: ${VNC_ENABLE:="NO"}
+: ${VNC_LISTEN:="127.0.0.1:5900"}
+: ${VNC_WIDTH:="1920"}
+: ${VNC_HEIGHT:="1080"}
+
+if [ "$VERBOSE" = "YES" ]; then
+    set -x
+fi
+
+############## Setup #########################
+
+function cleanup {
+    for vm in "${vms[@]}"; do
+        log "Destroying bhyve vm $vm"
+        bhyvectl "--vm=$vm" --destroy
+        log "Destroyed bhyve vm $vm"
+    done
+}
+vms=()
+for sig in EXIT; do
+  trap "set +e; sleep 10; cleanup" "$sig"
+done
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function log {
+    (>&2 echo "${@}")
+}
+
+############## Program #########################
+
+function main {
+    local cmd="$1"
+    shift 1
+    if [ "$cmd" = "create-disk" ]; then
+        create_disk "${@}"
+    elif [ "$cmd" = "start" ]; then
+        start_vm "${@}"
+    else
+        die 1 "Unrecognized command $cmd"
+    fi
+}
+
+function create_disk {
+    local zfs_path="$1"
+    local mount_path="$2"
+    local gigabytes="$3"
+    zfs create -o "mountpoint=$mount_path" "$zfs_path"
+    cp /usr/local/share/edk2-bhyve/BHYVE_UEFI_VARS.fd "${mount_path}/"
+    tee "${mount_path}/settings" <<EOF
+CPU_CORES="$CPU_CORES"
+MEMORY="$MEMORY"
+NETWORK="$NETWORK"
+IP_RANGE="$IP_RANGE"
+BRIDGE_NAME="$BRIDGE_NAME"
+INTERFACE_NAME="$INTERFACE_NAME"
+EOF
+    zfs create -s "-V${gigabytes}G" -o volmode=dev -o primarycache=metadata -o secondarycache=none -o volblocksize=64K "$zfs_path/disk0"
+}
+
+function start_vm {
+    local name="$1"
+    local zfs_path="$2"
+    local mount_path="$3"
+    local mount_cd="${4:-}"
+
+    if [ -e "${mount_path}/settings" ]; then
+        source "${mount_path}/settings"
+    fi
+
+    local host_interface_name="$INTERFACE_NAME" # for raw, external interface
+    local bridge_name="$BRIDGE_NAME"
+    local ip_range="$IP_RANGE" # for raw this value does not matter
+
+    local mac_address
+    mac_address=$(calculate_mac_address "$name")
+
+    local additional_args=()
+
+    if [ "$NETWORK" = "NAT" ]; then
+        assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        additional_args+=("-s" "2:0,e1000,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+    elif [ "$NETWORK" = "RAW" ]; then
+        assert_raw "$host_interface_name" "$bridge_name"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+    elif [ "$NETWORK" = "BOTH" ]; then
+        assert_bridge "jail_nat" "$bridge_name" "$ip_range"
+        assert_raw "$host_interface_name" "bridge_raw"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        local raw_bridge_link_name=$(detect_available_link "bridge_raw")
+        local raw_mac_address=$(calculate_mac_address "${name}_raw")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+        additional_args+=("-s" "3:0,virtio-net,netgraph,path=bridge_raw:,peerhook=${raw_bridge_link_name},mac=${raw_mac_address}")
+    elif [ "$NETWORK" = "NONE" ]; then
+        (>&2 echo "Not using any network.")
+    else
+        die 1 "Unrecognized NETWORK type $NETWORK"
+    fi
+
+
+    # -H release the CPU when guest issues HLT instruction. Otherwise 100% of core will be consumed.
+         # -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
+             # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
+            # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080 \
+
+    # TODO: Look into using nmdm instead of stdio for serial console
+    if [ -n "$mount_cd" ]; then
+        additional_args+=("-s" "5,ahci-cd,$mount_cd")
+    fi
+    if [ "$VNC_ENABLE" = "YES" ]; then
+        additional_args+=("-s" "29,fbuf,tcp=$VNC_LISTEN,w=$VNC_WIDTH,h=$VNC_HEIGHT,wait")
+    fi
+    vms+=("$name")
+    # Removes CPU_CORES because windows must be a single CPU in bhyve
+    # -c $CPU_CORES \
+        # We need tpm
+    #             -l "tpm,passthru,/dev/tpm0" \
+                    # -S \
+    while true; do
+        set -x
+        set +e
+        bhyve \
+            -D \
+            -c sockets=1,cores=2,threads=2 \
+            -m $MEMORY \
+            -H \
+            -w \
+            -o 'rtc.use_localtime=false' \
+            -s 0,hostbridge \
+            -s "4,nvme,/dev/zvol/${zfs_path}/disk0" \
+            -s 16,hda,play=/dev/dsp,rec=/dev/dsp \
+            -s 30,xhci,tablet \
+            -s 31,lpc -l com1,stdio \
+            -l "bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,${mount_path}/BHYVE_UEFI_VARS.fd" \
+            -U '5a63bcd1-5cb4-4401-8a6f-d4042fb928a6' \
+            "${additional_args[@]}" \
+            "$name"
+        local exit_code=$?
+        set -e
+        set +x
+        if [ $exit_code -eq 0 ]; then
+            echo "Rebooting."
+            sleep 5
+        elif [ $exit_code -eq 1 ]; then
+            echo "Powered off."
+            break
+        elif [ $exit_code -eq 2 ]; then
+            echo "Halted."
+            break
+        elif [ $exit_code -eq 3 ]; then
+            echo "Triple fault."
+            break
+        elif [ $exit_code -eq 4 ]; then
+            echo "Exited due to an error."
+            break
+        fi
+    done
+}
+
+function detect_available_link {
+    local bridge_name="$1"
+    local linknum=1
+    while true; do
+        local link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function assert_bridge {
+    local host_interface_name="$1"
+    local bridge_name="$2"
+    local ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function assert_raw {
+    local extif="$1"
+    local bridge_name="$2"
+
+    kldload -n ng_bridge ng_eiface ng_ether
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctlcat <<EOF
+# Create a bridge.
+mkpeer $extif: bridge lower link0
+# Assign a name to the bridge.
+name $extif:lower ${bridge_name}
+# Since the host is also using $extif, we need to connect the upper hook also. Otherwise we will lose connectivity.
+connect $extif: ${bridge_name}: upper link1
+
+# Enable promiscuous mode so the host ethernet adapter accepts packets for all addresses
+msg $extif: setpromisc 1
+
+# Do not overwrite source address on packets
+msg $extif: setautosrc 0
+EOF
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+function calculate_mac_address {
+    local name="$1"
+    local source
+    source=$(md5 -r -s "$name" | awk '{print $1}')
+    echo "06:${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}"
+}
+
+function find_available_port {
+    local start_port="$1"
+    local port="$start_port"
+    while true; do
+        sockstat -P tcp -p 443
+        port=$((port + 1))
+    done
+}
+
+function ngctlcat {
+    if [ "$VERBOSE" = "YES" ]; then
+        tee /dev/tty | ngctl -d -f -
+    else
+        ngctl -d -f -
+    fi
+}
+
+
+main "${@}"

ansible/roles/framework_laptop/files/screen_brightness_tmpfiles.conf

@@ -1,2 +1,2 @@
 # Set screen brightness. Ever since enabling adaptive brightness management, my brightness ends up sinking lower on re-boots (I suspect it is saving the actual brightness rather than the set brightness). This forces the brightness back to the level I prefer.
-w- /sys/class/backlight/amdgpu_bl0/brightness - - - - 85
+w- /sys/class/backlight/amdgpu_bl0/brightness - - - - 21845

ansible/roles/framework_laptop/files/wifi_us_modprobe.conf

@@ -0,0 +1 @@
+options cfg80211 ieee80211_regdom=US

ansible/roles/framework_laptop/files/windows

@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# REQUIRE: LOGIN
+# PROVIDE: windows
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+name=windows
+rcvar=${name}_enable
+start_cmd="${name}_start"
+stop_cmd="${name}_stop"
+status_cmd="${name}_status"
+load_rc_config $name
+
+tmux_name="windows"
+
+windows_start() {
+   /usr/local/bin/tmux new-session -d -s "$tmux_name" "/usr/bin/env VNC_ENABLE=YES VNC_LISTEN=0.0.0.0:5900 /usr/local/bin/bash /usr/local/bin/launch_windows start windows zroot/freebsd/current/vm/windows /vm/windows /vm/.iso/Win11_23H2_English_x64v2.iso"
+}
+
+windows_status() {
+    if /usr/local/bin/tmux has-session -t $tmux_name 2>/dev/null; then
+        echo "$tmux_name is running."
+    else
+        echo "$tmux_name is not running."
+        return 1
+    fi
+}
+
+windows_stop() {
+    /usr/local/bin/tmux has-session -t $tmux_name 2>/dev/null && (
+        /usr/local/bin/tmux kill-session -t $tmux_name
+        sleep 10
+        bhyvectl --vm=windows --destroy
+        # kill `cat /var/run/windows.pid`
+    )
+    windows_wait_for_end
+}
+
+windows_wait_for_end() {
+    while /usr/local/bin/tmux has-session -t $tmux_name 2>dev/null; do
+        sleep 1
+    done
+}
+
+run_rc_command "$1"

ansible/roles/framework_laptop/meta/main.yaml

@@ -0,0 +1,3 @@
+dependencies:
+  - role: bhyve
+    when: 'os_flavor == "freebsd"'

ansible/roles/framework_laptop/tasks/freebsd.yaml

@@ -1,5 +1,30 @@
-# - name: Install packages
-#   package:
-#     name:
-#       - foo
-#     state: present
+- name: Install loader.conf
+  copy:
+    src: "files/{{ item }}_loader.conf"
+    dest: "/boot/loader.conf.d/{{ item }}.conf"
+    mode: 0644
+    owner: root
+    group: wheel
+  loop:
+    - disable_wifi_powersave
+
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: launch_windows.bash
+      dest: /usr/local/bin/launch_windows
+
+- name: Install rc script
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}"
+    owner: root
+    group: wheel
+    mode: 0755
+  loop:
+    - src: windows

ansible/roles/framework_laptop/tasks/linux.yaml

@@ -30,10 +30,11 @@
     - iwlwifi
     - snd_hda_intel
     - disable_sp5100_watchdog
+    - wifi_us
 
 - name: Configure kernel command line
   zfs:
-    name: "zroot/linux"
+    name: "zroot/linux/archwork/be"
     state: present
     extra_zfs_properties:
       # amdgpu.abmlevel=3 :: Automatically reduce screen brightness but tweak colors to compensate for power reduction.
@@ -42,7 +43,8 @@
       # amd_pstate=passive :: Fully automated hardware pstate control.
       # amd_pstate=active :: Same as passive except we can set the energy performance preference (EPP) to suggest how much we prefer performance or energy efficiency.
       # amd_pstate=guided :: Same as passive except we can set upper and lower frequency bounds.
-      "org.zfsbootmenu:commandline": "rw quiet amdgpu.abmlevel=3 pcie_aspm=force pcie_aspm.policy=powersupersave nowatchdog"
+      # amdgpu.dcdebugmask=0x10 :: Allegedly disables Panel Replay from https://community.frame.work/t/tracking-freezing-arch-linux-amd/39495/32
+      "org.zfsbootmenu:commandline": "rw quiet amdgpu.abmlevel=2 pcie_aspm=force pcie_aspm.policy=powersupersave nowatchdog amdgpu.dcdebugmask=0x10"
 
 - name: Install Configuration
   copy:
@@ -65,3 +67,34 @@
   loop:
     - gpe10-boot.service
     - gpe10-sleep.service
+# install swtpm
+# install edk2-ovmf for /usr/share/ovmf/OVMF.fd
+# install qemu-system-x86
+
+# doas qemu-system-x86_64 -cdrom /vm/.iso/Win11_23H2_English_x64v2.iso -cpu Skylake-Client-v3 -enable-kvm -m 8192 —device chardev,socket,id=chrtpm,path=/tmp/emulated_tpm/swtpm-sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0 -smp 2 -device intel-hda -device hda-duplex -usb -nic user,ipv6=off,model=rtl8139,mac=84:1b:77:c9:03:a6 -bios /usr/share/edk2/x64/OVMF.fd -drive file=/dev/zvol/zroot/freebsd/current/vm/windows/disk0,format=raw,media=disk,if=none,id=nvm -device nvme,drive=nvm,serial=foo,opt_io_size=4096,min_io_size=4096,logical_block_size=4096,physical_block_size=4096
+
+# doas mkdir /tmp/emulated_tpm
+# doas swtpm socket --tpmstate dir=/tmp/emulated_tpm --ctrl type=unixio,path=/tmp/emulated_tpm/swtpm-sock --log level=20 --tpm2
+
+- name: Build aur packages
+  register: buildaur
+  become_user: "{{ build_user.name }}"
+  command: "aurutils-sync --no-view {{ item }}"
+  args:
+    creates: "/var/cache/pacman/custom/{{ item }}-*.pkg.tar.*"
+  loop:
+    - fw-ectool-git
+
+- name: Update cache
+  when: buildaur.changed
+  pacman:
+    name: []
+    state: present
+    update_cache: true
+
+- name: Install packages
+  package:
+    name:
+      - fw-ectool-git
+      - wireless-regdb
+    state: present

ansible/roles/gpg/files/gpg.asc

@@ -1,27 +1,27 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
-mDMEXZwWGhYJKwYBBAHaRw8BAQdAfv7qozKkmf4D+5PDzADsMm4aAKDGLha7+Cu0
-0H+RsWG0HlRvbSBBbGV4YW5kZXIgPHdvcmtAZml6ei5idXp6PoiQBBMWCAA4FiEE
-uEgVk2PCh3kXlUvhJ95A2bhFXBsFAl+w+R0CGwMFCwkIBwIGFQoJCAsCBBYCAwEC
-HgECF4AACgkQJ95A2bhFXBt6fgD+NOYnw9gz5K/q3H5LE/JvqzCSHezJmeGgif0C
-uU4m1/MA+gPDKME7syEtJsTpELEMrxWWpDW0tD/W1iJE7roGYPQPtB1Ub20gQWxl
-eGFuZGVyIDx0b21AZml6ei5idXp6PoiQBBMWCAA4FiEEuEgVk2PCh3kXlUvhJ95A
-2bhFXBsFAl2cFhoCGwMFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQJ95A2bhF
-XBvYJQEA19wc2s/bEKcnHONC3i8UikLFqZXyYoH6/MFjoAteU8sBAKpE7Qq0zbJb
-XWRESzK3u6p7/+kUqOeDltAuKXTe1FAGuDMEXZwWyhYJKwYBBAHaRw8BAQdAPyIL
-4EGg4T5JO9q2kpVDy2WjMiXz3nZXwYW4GLoTYkiI9QQYFggAJgIbAhYhBLhIFZNj
-wod5F5VL4SfeQNm4RVwbBQJlC4ZhBQkLMdaXAIF2IAQZFggAHRYhBIHmRDmWdVAu
-sSUWutOhecmlPA7eBQJdnBbKAAoJENOhecmlPA7ejJ4A/iq7N2mMhx+ovOXm1REo
-ASPF3l4YAAjOHsXqcPtFHKGJAQCiuA71d6CQ+qNZLuka/KVB/etkkJvDzvaTtiQQ
-QG+gAwkQJ95A2bhFXBtRDgEAqymMavroD5c/4+M/EZ3/d8wxfA9E3Fb/1mt4c2Zr
-NnkBAKYOM+pz/pncFnV+kF7h7TQEEYuGw1JhJVT/duA4lwsLuDMEXZwXARYJKwYB
-BAHaRw8BAQdAa76TmWuKuiR1bnNV1FUE6oQ4C8A+UiQb8x0k1z2DmTKIfgQYFggA
-JgIbIBYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZwBQkLMdZgAAoJECfeQNm4
-RVwb8TkA/RkBu9Ev8iDE5nvn8YF8FRiY56Z5d+SBPG4VvrCzXrmlAP46wUjIRpkM
-rTbb1GMbvYnkeOrBs/qiWjEtHHc3ZLMWD7g4BF2cFygSCisGAQQBl1UBBQEBB0AO
-0t3BUxLuokTqKVcheFAZd4UKxAGznPQlvsVyhWWIEgMBCAeIfgQYFggAJgIbDBYh
-BLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZwBQkLMdY5AAoJECfeQNm4RVwbXscA
-/A8zRRTCwQKxJ8iz5jmTcVFAhl2vD781Dtv8NvcWd5t8APwIwcuFVZZA3yayhIxi
-3aqYpMRxpn2t6Nswax1MIM8DBQ==
-=dzEV
+mDMEaNLjzBYJKwYBBAHaRw8BAQdAoegj6iXzJgxBkW8LyRS8ANRzp0LqyFbW1kRr
+Z4VtVRK0HlRvbSBBbGV4YW5kZXIgPHdvcmtAZml6ei5idXp6PoiQBBMWCAA4FiEE
+0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7GoCGwEFCwkIBwIGFQoJCAsCBBYCAwEC
+HgECF4AACgkQQngpn7hPaHXNRAEAxOHPULwbf/FIzS7spmdSYrcCX/foaB78rpCT
+/MzDPvMBANy0PcseR1ZxoHZDcAsYDa0CSCrO6oLwPFriVss3RA0GtB1Ub20gQWxl
+eGFuZGVyIDx0b21AZml6ei5idXp6PoiTBBMWCAA7AhsBBQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7MkCGQEACgkQQngp
+n7hPaHX9fQEA4ngwEKr0nlKxH5bQV9u/EJeI3wbSgBjlnyTQuI79AB4BAO6+frGt
+8S+p6qFZ4ufqyGPfklxPeOJLSYk0PLKVNMcHuDMEaNLm8xYJKwYBBAHaRw8BAQdA
+HDhppS6yD8j1Bb/i6ku16uQ3qhshDNA9cOQeMxBae9aI9QQYFggAJhYhBNJyyNYW
+fyaFlGdmb0J4KZ+4T2h1BQJo0ubzAhsCBQkDwmcAAIEJEEJ4KZ+4T2h1diAEGRYI
+AB0WIQS9v3ap15pUELURqaY2yZ6LPDnYXwUCaNLm8wAKCRA2yZ6LPDnYXyaNAPsF
+gR37jEqfgEByVsoKY6bB82T79o9d4FQe1iPsURyuLwD/fkQyV3NwGjysxkoZqYmK
+mXJYqtWRBTe2G2UUkm6E/QafHwD+IbkCZ6sGTcexsqzex5x6U8TOvbdVS4dKjSf1
+nVRGxvwBAPiIJsXWVuwmskWMDpcaW/qgQ8hOEuq7/vlkZDGOnMgOuDgEaNLnDBIK
+KwYBBAGXVQEFAQEHQBcOCDGnrRwv51c5B7QVLMkLC2UKUzPPrahLZHT3RWhmAwEI
+B4h+BBgWCAAmFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS5wwCGwwFCQPCZwAA
+CgkQQngpn7hPaHUZIAD/ZwQ9sLIwuO5qPFAAkqcaNyt68O6WkD8sKaq1r/TPviAA
+/j92d7cRUIkJtS8odRYlK51r9eMeTGh2npaO+j3VKCgBuDMEaNLnJRYJKwYBBAHa
+Rw8BAQdAPT7jOLbozd5hacityJHniQ6UbHN+AJcb6jh5rXOnOuSIfgQYFggAJhYh
+BNJyyNYWfyaFlGdmb0J4KZ+4T2h1BQJo0uclAhsgBQkDwmcAAAoJEEJ4KZ+4T2h1
+rREA/3QE6suVUDl4OS2tCi4z2fh/7kjt29I3IFo+/B0AOumgAP0ao8FGqJyFC8YA
+7V6T4qrXHbhlqTeofGhQ+iu7HqZVCw==
+=OfDR
 -----END PGP PUBLIC KEY BLOCK-----

ansible/roles/gpg/files/gpg_work.asc

@@ -1,27 +1,27 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
-mDMEXZwWGhYJKwYBBAHaRw8BAQdAfv7qozKkmf4D+5PDzADsMm4aAKDGLha7+Cu0
-0H+RsWG0LVRob21hcyBBbGV4YW5kZXIgPFRob21hc0EuQWxleGFuZGVyQGhtaG4u
-b3JnPoiQBBMWCAA4FiEEuEgVk2PCh3kXlUvhJ95A2bhFXBsFAmULicsCGwMFCwkI
-BwIGFQoJCAsCBBYCAwECHgECF4AACgkQJ95A2bhFXBsUtQD9GWPdWc/nSmO0Gp7p
-DzxrieliriAnO+ZCHp31mFbMtToBAPxPYN9y4kgSiXhLiFLoRK5k5FCspksTSitg
-0CbXDE4LuDgEXZwWGhIKKwYBBAGXVQEFAQEHQK202EIAwTBuxARUygOvn+AloMJd
-ui39m+nMghn1MNo+AwEIB4h4BBgWCAAgFiEEuEgVk2PCh3kXlUvhJ95A2bhFXBsF
-Al2cFhoCGwwACgkQJ95A2bhFXBtNzAEAq5I6xPjIbb23xmhxh5cM/UJxdGedfWMy
-vF6/JtDvtPUBAPQRQn5AMwTOA+CSnliYf7ZjfVOlHscy60XWPlvXLoAJuDMEXZwW
-yhYJKwYBBAHaRw8BAQdAPyIL4EGg4T5JO9q2kpVDy2WjMiXz3nZXwYW4GLoTYkiI
-9QQYFggAJgIbAhYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZhBQkLMdaXAIF2
-IAQZFggAHRYhBIHmRDmWdVAusSUWutOhecmlPA7eBQJdnBbKAAoJENOhecmlPA7e
-jJ4A/iq7N2mMhx+ovOXm1REoASPF3l4YAAjOHsXqcPtFHKGJAQCiuA71d6CQ+qNZ
-Luka/KVB/etkkJvDzvaTtiQQQG+gAwkQJ95A2bhFXBtRDgEAqymMavroD5c/4+M/
-EZ3/d8wxfA9E3Fb/1mt4c2ZrNnkBAKYOM+pz/pncFnV+kF7h7TQEEYuGw1JhJVT/
-duA4lwsLuDMEXZwXARYJKwYBBAHaRw8BAQdAa76TmWuKuiR1bnNV1FUE6oQ4C8A+
-UiQb8x0k1z2DmTKIfgQYFggAJgIbIBYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJl
-C4ZwBQkLMdZgAAoJECfeQNm4RVwb8TkA/RkBu9Ev8iDE5nvn8YF8FRiY56Z5d+SB
-PG4VvrCzXrmlAP46wUjIRpkMrTbb1GMbvYnkeOrBs/qiWjEtHHc3ZLMWD7g4BF2c
-FygSCisGAQQBl1UBBQEBB0AO0t3BUxLuokTqKVcheFAZd4UKxAGznPQlvsVyhWWI
-EgMBCAeIfgQYFggAJgIbDBYhBLhIFZNjwod5F5VL4SfeQNm4RVwbBQJlC4ZwBQkL
-MdY5AAoJECfeQNm4RVwbXscA/A8zRRTCwQKxJ8iz5jmTcVFAhl2vD781Dtv8NvcW
-d5t8APwIwcuFVZZA3yayhIxi3aqYpMRxpn2t6Nswax1MIM8DBQ==
-=0HtE
+mDMEaNLjzBYJKwYBBAHaRw8BAQdAoegj6iXzJgxBkW8LyRS8ANRzp0LqyFbW1kRr
+Z4VtVRK0HlRvbSBBbGV4YW5kZXIgPHdvcmtAZml6ei5idXp6PoiQBBMWCAA4FiEE
+0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7GoCGwEFCwkIBwIGFQoJCAsCBBYCAwEC
+HgECF4AACgkQQngpn7hPaHXNRAEAxOHPULwbf/FIzS7spmdSYrcCX/foaB78rpCT
+/MzDPvMBANy0PcseR1ZxoHZDcAsYDa0CSCrO6oLwPFriVss3RA0GtB1Ub20gQWxl
+eGFuZGVyIDx0b21AZml6ei5idXp6PoiTBBMWCAA7AhsBBQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS7MkCGQEACgkQQngp
+n7hPaHX9fQEA4ngwEKr0nlKxH5bQV9u/EJeI3wbSgBjlnyTQuI79AB4BAO6+frGt
+8S+p6qFZ4ufqyGPfklxPeOJLSYk0PLKVNMcHuDMEaNLm8xYJKwYBBAHaRw8BAQdA
+HDhppS6yD8j1Bb/i6ku16uQ3qhshDNA9cOQeMxBae9aI9QQYFggAJhYhBNJyyNYW
+fyaFlGdmb0J4KZ+4T2h1BQJo0ubzAhsCBQkDwmcAAIEJEEJ4KZ+4T2h1diAEGRYI
+AB0WIQS9v3ap15pUELURqaY2yZ6LPDnYXwUCaNLm8wAKCRA2yZ6LPDnYXyaNAPsF
+gR37jEqfgEByVsoKY6bB82T79o9d4FQe1iPsURyuLwD/fkQyV3NwGjysxkoZqYmK
+mXJYqtWRBTe2G2UUkm6E/QafHwD+IbkCZ6sGTcexsqzex5x6U8TOvbdVS4dKjSf1
+nVRGxvwBAPiIJsXWVuwmskWMDpcaW/qgQ8hOEuq7/vlkZDGOnMgOuDgEaNLnDBIK
+KwYBBAGXVQEFAQEHQBcOCDGnrRwv51c5B7QVLMkLC2UKUzPPrahLZHT3RWhmAwEI
+B4h+BBgWCAAmFiEE0nLI1hZ/JoWUZ2ZvQngpn7hPaHUFAmjS5wwCGwwFCQPCZwAA
+CgkQQngpn7hPaHUZIAD/ZwQ9sLIwuO5qPFAAkqcaNyt68O6WkD8sKaq1r/TPviAA
+/j92d7cRUIkJtS8odRYlK51r9eMeTGh2npaO+j3VKCgBuDMEaNLnJRYJKwYBBAHa
+Rw8BAQdAPT7jOLbozd5hacityJHniQ6UbHN+AJcb6jh5rXOnOuSIfgQYFggAJhYh
+BNJyyNYWfyaFlGdmb0J4KZ+4T2h1BQJo0uclAhsgBQkDwmcAAAoJEEJ4KZ+4T2h1
+rREA/3QE6suVUDl4OS2tCi4z2fh/7kjt29I3IFo+/B0AOumgAP0ao8FGqJyFC8YA
+7V6T4qrXHbhlqTeofGhQ+iu7HqZVCw==
+=OfDR
 -----END PGP PUBLIC KEY BLOCK-----

ansible/roles/graphics/tasks/freebsd_intel.yaml

@@ -8,7 +8,6 @@
       - libva-utils # for vainfo
       - vdpauinfo # for vdpauinfo
       - libvdpau-va-gl # vdpau support
-      - igt-gpu-tools # for intel_gpu_top
       - vulkan-tools # For vulkaninfo
     state: present
 

ansible/roles/jail/files/fstab_bastion

@@ -1,4 +1,4 @@
 tmpfs /jail/bastion/tmp tmpfs rw,mode=777 0 0
 tmpfs /jail/bastion/var/run tmpfs rw,mode=755 0 0
 
-/jail/certificate/usr/local/etc/letsencrypt/archive/stuff.fizz.buzz       /jail/bastion/stuff.fizz.buzz     nullfs    ro,noexec     0      0
+/jail/certificate/usr/local/etc/letsencrypt       /jail/bastion/letsencrypt     nullfs    ro,noexec     0      0

ansible/roles/jail/files/jail_netgraph_bridge.bash

@@ -23,11 +23,15 @@ function start_jail {
     jail_interface_name=$(sanitize_interface_name "$2")
     ip_range="$3"
 
+    local mac_address
+    mac_address=$(calculate_mac_address "$jail_interface_name")
+
     assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
 
     bridge_link_name=$(detect_available_link "${bridge_name}")
     ngctl -d -f - <<EOF
 mkpeer ${bridge_name}: eiface $bridge_link_name ether
+msg ${bridge_name}:$bridge_link_name set $mac_address
 name ${bridge_name}:$bridge_link_name $jail_interface_name
 EOF
     ifconfig $(ngctl msg "${jail_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${jail_interface_name}" up
@@ -121,4 +125,11 @@ function sanitize_interface_name {
     echo "${1:0:15}"
 }
 
+function calculate_mac_address {
+    local name="$1"
+    local source
+    source=$(md5 -r -s "$name" | awk '{print $1}')
+    echo "06:${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}"
+}
+
 main "${@}"

ansible/roles/jail/files/jails/admin_git.conf

@@ -2,7 +2,7 @@ admin_git {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/files/jails/bastion.conf

@@ -2,7 +2,7 @@ bastion {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/files/jails/certificate.conf

@@ -2,7 +2,7 @@ certificate {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/files/jails/cloak.conf

@@ -4,8 +4,8 @@ cloak {
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start restricted_nat jail${name} 10.215.2.1/24";
     # Create a dummy interface that is never used, just to create the cloak bridge that is used by children.
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start cloak dummy${name} 192.168.1.0/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop cloak dummy{name}";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop restricted_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop cloak dummy{name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop restricted_nat jail${name}";
     vnet.interface += "jail${name}";
     vnet.interface += "cloak";
 

ansible/roles/jail/files/jails/dagger.conf

@@ -4,8 +4,10 @@ dagger {
     vnet.interface += "dagger";
 
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start cloak ${name} 192.168.1.0/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop cloak ${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop cloak ${name}";
 
+    devfs_ruleset = 15;
+    mount.devfs;
     mount.fstab = "/etc/fstab.${name}";
 
     exec.start += "/bin/sh /etc/rc";

ansible/roles/jail/files/jails/momlaptop.conf

@@ -0,0 +1,15 @@
+momlaptop {
+    path = "/jail/${name}";
+    vnet;
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
+    devfs_ruleset = 14;
+    mount.devfs;
+    mount.fstab = "/etc/fstab.${name}";
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}

ansible/roles/jail/files/jails/nat_dhcp.conf

@@ -2,7 +2,7 @@ nat_dhcp {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/files/jails/olddagger.conf

@@ -4,7 +4,7 @@ olddagger {
     vnet.interface += "olddagger";
 
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start cloak ${name} 192.168.1.0/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop cloak ${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop cloak ${name}";
 
     mount.fstab = "/etc/fstab.${name}";
 

ansible/roles/jail/files/jails/public_dns.conf

@@ -2,7 +2,7 @@ public_dns {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/files/jails/rg.conf

@@ -0,0 +1,15 @@
+rg {
+    path = "/jail/${name}";
+    vnet;
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
+    devfs_ruleset = 14;
+    mount.devfs;
+    mount.fstab = "/etc/fstab.${name}";
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}

ansible/roles/jail/files/jails/sample.conf

@@ -2,7 +2,7 @@ sample {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/files/jails/sftp.conf

@@ -2,7 +2,7 @@ sftp {
     path = "/jail/${name}";
     vnet;
     exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
-    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    exec.poststop += "sleep 10; /usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
     vnet.interface += "jail${name}";
 
     devfs_ruleset = 14;

ansible/roles/jail/templates/new_jail.bash.j2

@@ -26,7 +26,7 @@ function by_src {
 }
 
 function by_bin {
-    DESTRELEASE=13.2-RELEASE
+    DESTRELEASE=14.3-RELEASE
     DESTARCH=`uname -m`
     SOURCEURL=http://ftp.freebsd.org/pub/FreeBSD/releases/$DESTARCH/$DESTRELEASE/
     for component in base ports; do fetch $SOURCEURL/$component.txz -o - | tar -xf - -C "$DESTDIR" ; done

ansible/roles/jail_bastion/files/nginx.conf

@@ -36,8 +36,8 @@ http {
 
         include conf.d/tls_settings.include;
         # RSA
-        ssl_certificate /stuff.fizz.buzz/fullchain1.pem;
-        ssl_certificate_key /stuff.fizz.buzz/privkey1.pem;
+        ssl_certificate /letsencrypt/live/stuff.fizz.buzz/fullchain.pem;
+        ssl_certificate_key /letsencrypt/live/stuff.fizz.buzz/privkey.pem;
 
         # Nginx by default only allows file uploads up to 1M in size
         client_max_body_size 50M;

ansible/roles/jail_bastion/tasks/freebsd.yaml

@@ -17,7 +17,7 @@
     owner: root
     group: wheel
   loop:
-    - /stuff.fizz.buzz
+    - /letsencrypt
     - /etc/rc.conf.d
     - /usr/local/etc/nginx/conf.d
 

ansible/roles/jail_momlaptop/files/headers.include

@@ -0,0 +1,15 @@
+# Enable HTTP Strict Transport Security (HSTS) to force clients to
+# always connect via HTTPS (do not use if only testing)
+add_header Strict-Transport-Security "max-age=31536000;" always;
+# Enable cross-site filter (XSS) and tell browser to block detected
+# attacks
+add_header X-XSS-Protection "1; mode=block" always;
+# Prevent some browsers from MIME-sniffing a response away from the
+# declared Content-Type
+add_header X-Content-Type-Options "nosniff" always;
+# Disallow the site to be rendered within a frame (clickjacking
+# protection)
+add_header X-Frame-Options "DENY" always;
+
+# Indicate that we are serving http3 on port 443
+add_header Alt-Svc 'h3=":8033"; ma=864000';

ansible/roles/jail_momlaptop/files/newsyslog.conf

@@ -0,0 +1,2 @@
+# logfilename          [owner:group]    mode count size when  flags [/pid_file] [sig_num]
+/var/log/nginx/*.log			640  5	   1000	@T00 GYC /var/run/nginx.pid SIGUSR1

ansible/roles/jail_momlaptop/files/nginx.conf

@@ -0,0 +1,48 @@
+worker_processes  auto;
+user  www www;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    types {
+        text/plain log;
+    }
+
+    sendfile        on;
+    tcp_nopush     on;
+    tcp_nodelay    on;
+    gzip  on;
+
+    include conf.d/headers.include;
+
+    server {
+        listen 443 quic reuseport;
+        listen [::]:443 quic reuseport;
+        listen 443 ssl;
+        listen [::]:443 ssl;
+        http2  on;
+
+        server_name momlaptop.fizz.buzz;
+
+        include conf.d/tls_settings.include;
+        # RSA
+        ssl_certificate /momlaptop.fizz.buzz/tls.crt;
+        ssl_certificate_key /momlaptop.fizz.buzz/tls.key;
+
+        # Nginx by default only allows file uploads up to 50M in size
+        client_max_body_size 50M;
+
+        location / {
+            auth_basic           "Stuff";
+            auth_basic_user_file conf.d/htpasswd;
+
+            alias /srv/http/;
+            autoindex on;
+        }
+    }
+}

ansible/roles/jail_momlaptop/files/nginx_rc.conf

@@ -0,0 +1 @@
+nginx_enable="YES"