Merge branch 'bhyve'

ansible/environments/laptop/host_vars/odofreebsd

@@ -44,3 +44,5 @@ jail_list:
   - name: dagger
     conf:
       src: dagger
+bhyve_dataset: zroot/freebsd/release/vm
+bhyve_list: []

ansible/playbook.yaml

@@ -27,3 +27,4 @@
     - fuse
     - autofs
     - exfat
+    - bhyve

ansible/roles/base/files/bemount.bash

@@ -0,0 +1,126 @@
+#!/usr/local/bin/bash
+#
+# Mount non-boot-environment datasets.
+#
+# We can't leave datasets outside the boot environment (for example,
+# jails or bhyve VMs) as canmount=on because then every boot
+# environment's external datasets would all attempt to mount every
+# time. To work around this, we mark those datasets as canmount=noauto
+# and run this script to mount datasets under the root of our boot
+# environment. This script depends heavily on my zfs dataset structure
+# so it needs to be improved to be robust enough for different
+# layouts. An example of my layout is:
+#
+## NAME                                                                                               MOUNTPOINT                  CANMOUNT  TA:BEMOUNT
+## zroot                                                                                              none                        off       -
+## zroot/global                                                                                       /global                     on        -
+## zroot/freebsd                                                                                      none                        on        -
+## zroot/freebsd/13.1-RELEASE                                                                         none                        on        -
+## zroot/freebsd/13.1-RELEASE/be                                                                      none                        on        -
+## zroot/freebsd/13.1-RELEASE/be/main                                                                 /                           noauto    -
+## zroot/freebsd/13.1-RELEASE/jails                                                                   none                        on        -
+## zroot/freebsd/13.1-RELEASE/jails/foo                                                               /jail/foo                   noauto    on
+## zroot/freebsd/13.1-RELEASE/jails/bar                                                               /jail/bar                   noauto    on
+## zroot/freebsd/13.1-RELEASE/jails/baz                                                               /jail/baz                   noauto    on
+## zroot/freebsd/13.1-RELEASE/vm-bhyve                                                                /vm                         noauto    on
+## zroot/linux                                                                                        none                        on        -
+## zroot/linux/arch                                                                                   none                        on        -
+## zroot/linux/arch/be                                                                                none                        on        -
+## zroot/linux/arch/be/main                                                                           /                           noauto    -
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    local all_zfs_datasets=$(zfs list -Hp -o 'name,mountpoint,canmount,ta:bemount,mounted')
+    local root_dataset=$(find_root_dataset "$all_zfs_datasets")
+    local datasets_to_mount=$(find_datasets_to_mount_for_boot_environment "$all_zfs_datasets" "$root_dataset")
+    if [ -n "$datasets_to_mount" ]; then
+        mount_datasets "$datasets_to_mount"
+    fi
+}
+
+function reverse_lines {
+    sed '1!x;H;1h;$!d;g'
+}
+
+function find_dataset {
+    local all_zfs_datasets="$1"
+    local dataset_name="$2"
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        if [ "$ds_name" = "$dataset_name" ]; then
+            echo "$dataset"
+            return
+        fi
+    done<<<"$all_zfs_datasets"
+}
+
+function find_root_dataset {
+    local all_zfs_datasets="$1"
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        local ds_mountpoint=$(awk '{print $2}'<<<"$dataset")
+        # local ds_canmount=$(awk '{print $3}'<<<"$dataset")
+        # local ds_bemount=$(awk '{print $4}'<<<"$dataset")
+        local ds_mounted=$(awk '{print $5}'<<<"$dataset")
+        if [ "$ds_mounted" = "yes" ] && [ "$ds_mountpoint" = "/" ]; then
+            echo "$ds_name"
+            return
+        fi
+    done<<<"$all_zfs_datasets"
+}
+
+function find_datasets_to_mount_for_boot_environment {
+    local all_zfs_datasets="$1"
+    local root_dataset="$2"
+    # This is a consequence of my layout for zfs datasets. I should
+    # make this more robust. Perhaps a zfs property like search up
+    # from dataset mounted at / until you find a dataset with property
+    # ta:bemountroot="on"?
+    local be_root_name="${root_dataset%/*/*}"
+    local be_root_dataset=$(find_dataset "$all_zfs_datasets" "$be_root_name")
+
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        # local ds_mountpoint=$(awk '{print $2}'<<<"$dataset")
+        local ds_canmount=$(awk '{print $3}'<<<"$dataset")
+        local ds_bemount=$(awk '{print $4}'<<<"$dataset")
+        local ds_mounted=$(awk '{print $5}'<<<"$dataset")
+
+        case "$ds_name" in
+            "${be_root_name}/"*) ;;
+            *)    continue ;;
+        esac
+
+        if [ "$ds_bemount" != "on" ]; then
+            continue
+        fi
+
+        if [ "$ds_mounted" != "no" ]; then
+            continue
+        fi
+
+        if [ "$ds_canmount" != "noauto" ]; then
+            continue
+        fi
+        echo "$dataset"
+
+    done<<<"$all_zfs_datasets"
+}
+
+function mount_datasets {
+    local datasets_to_mount=$(reverse_lines<<<"$1")
+    while read dataset; do
+        local ds_name=$(awk '{print $1}'<<<"$dataset")
+        local ds_mountpoint=$(awk '{print $2}'<<<"$dataset")
+        local ds_canmount=$(awk '{print $3}'<<<"$dataset")
+        local ds_bemount=$(awk '{print $4}'<<<"$dataset")
+        local ds_mounted=$(awk '{print $5}'<<<"$dataset")
+
+        mount -v -t zfs "$ds_name" "$ds_mountpoint"
+
+    done<<<"$datasets_to_mount"
+}
+
+main

ansible/roles/base/files/bemount_rc.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+#
+# REQUIRE: FILESYSTEM kld
+# PROVIDE: bemount
+
+. /etc/rc.subr
+name=bemount
+rcvar=${name}_enable
+start_cmd="${name}_start"
+stop_cmd="${name}_stop"
+load_rc_config $name
+
+bemount_start() {
+    /usr/local/bin/bemount
+}
+
+bemount_stop() {
+}
+
+run_rc_command "$1"

ansible/roles/base/tasks/freebsd.yaml

@@ -94,3 +94,31 @@
       src: tmpfs
       fstype: tmpfs
       opts: rw,mode=777
+
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: bemount.bash
+      dest: /usr/local/bin/bemount
+
+- name: Install rc script
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}"
+    owner: root
+    group: wheel
+    mode: 0755
+  loop:
+    - src: bemount_rc.sh
+      dest: bemount
+
+- name: Enable bemount
+  community.general.sysrc:
+    name: bemount_enable
+    value: "YES"
+    path: /etc/rc.conf.d/bemount

ansible/roles/bhyve/defaults/main.yaml

@@ -0,0 +1,2 @@
+bhyve_mountpoint: "/vm"
+bhyve_list: []

ansible/roles/bhyve/files/arch.conf

@@ -0,0 +1,22 @@
+# Use UEFI when booting from a disk
+loader="uefi"
+graphics="yes"
+xhci_mouse="yes"
+# graphics_listen="0.0.0.0"
+graphics_listen="127.0.0.1"
+# graphics_listen="10.216.1.1"
+graphics_res="1920x1080"
+# graphics_wait="yes"
+# prestart=""
+console="tmux"
+
+cpu=1
+memory=1024M
+
+disk0_type="virtio-blk"
+disk0_name="disk0"
+disk0_dev="sparse-zvol"
+virt_random="yes"  # virtio-rnd
+
+# Creates a link to host_bridge1's link3 hook to the vmlink hook on a type socket
+bhyve_options="-s 2:0,virtio-net,netgraph,path=host_bridge1:,peerhook=link3"

ansible/roles/bhyve/tasks/common.yaml

@@ -0,0 +1,14 @@
+- import_tasks: tasks/freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/linux.yaml
+  when: 'os_flavor == "linux"'
+
+- include_tasks:
+    file: tasks/peruser.yaml
+    apply:
+      become: yes
+      become_user: "{{ initialize_user }}"
+  loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}"
+  loop_control:
+    loop_var: initialize_user

ansible/roles/bhyve/tasks/freebsd.yaml

@@ -0,0 +1,54 @@
+#
+# Create a new VM:
+#   vm iso 'http://mirror.clarkson.edu/archlinux/iso/2022.01.01/archlinux-2022.01.01-x86_64.iso'
+#   vm create -t arch -s 50G testvm
+#   vm install testvm 'archlinux-2022.01.01-x86_64.iso'
+#
+#
+- name: Install packages
+  package:
+    name:
+      - vm-bhyve
+      - tmux # for interactive consoles
+      - bhyve-firmware # For UEFI
+    state: present
+
+- name: Create zfs dataset
+  zfs:
+    name: "{{ bhyve_dataset }}"
+    state: present
+    extra_zfs_properties:
+      mountpoint: "{{ bhyve_mountpoint }}"
+      canmount: "noauto"
+      "ta:bemount": "on"
+
+- name: Enable bhyve
+  community.general.sysrc:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    path: /etc/rc.conf.d/vm
+  loop:
+    - name: vm_enable
+      value: "YES"
+    - name: vm_dir
+      value: "zfs:{{ bhyve_dataset }}"
+    - name: vm_list
+      value: "{{ bhyve_list|community.general.json_query('[?enabled==`true`].name')|join(' ') }}"
+    - name: vm_delay
+      value: "5"
+
+- name: init vm-bhyve
+  command: vm init
+  args:
+    creates: "{{ bhyve_mountpoint }}/.templates"
+
+- name: Install Configuration
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0644
+    owner: root
+    group: wheel
+  loop:
+    - src: arch.conf
+      dest: "{{ bhyve_mountpoint }}/.templates/arch.conf"

ansible/roles/bhyve/tasks/linux.yaml

@@ -0,0 +1,6 @@
+# - name: Install packages
+#   pacman:
+#     name:
+#       - foo
+#     state: present
+#     update_cache: true

ansible/roles/bhyve/tasks/main.yaml

@@ -0,0 +1,2 @@
+- import_tasks: tasks/common.yaml
+  when: os_flavor == "freebsd" and bhyve_dataset is defined

ansible/roles/bhyve/tasks/peruser.yaml

@@ -0,0 +1,29 @@
+- include_role:
+    name: per_user
+
+# - name: Create directories
+#   file:
+#     name: "{{ account_homedir.stdout }}/{{ item }}"
+#     state: directory
+#     mode: 0700
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - ".config/foo"
+
+# - name: Copy files
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ account_homedir.stdout }}/{{ item.dest }}"
+#     mode: 0600
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - src: foo.conf
+#       dest: .config/foo/foo.conf
+
+- import_tasks: tasks/peruser_freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/peruser_linux.yaml
+  when: 'os_flavor == "linux"'

ansible/roles/firewall/files/odofreebsd_pf.conf

@@ -41,4 +41,5 @@ pass in on $ext_if proto udp to any port $udp_pass_in
 pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
 
 pass in on host_uplink0 proto udp from any to any port { 53 51820 }
+pass out on host_uplink0 proto tcp from any to any port 8081
 pass in on host_uplink1

ansible/roles/jail/tasks/freebsd.yaml

@@ -10,7 +10,7 @@
   zfs:
     name: "{{ item.dataset|default(jail_zfs_dataset) }}/jails/{{ item.name }}"
     state: present
-    extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({})) }}'
+    extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({}))|combine({''canmount'': ''noauto'', ''ta:bemount'': ''on''}) }}'
 
   loop: "{{ jail_list }}"
 
@@ -27,7 +27,7 @@
   zfs:
     name: "{{ item.0.dataset|default(jail_zfs_dataset) }}/persistent/{{ item.0.name }}/{{ item.1.name }}"
     state: present
-    extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({})) }}'
+    extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({}))|combine({''canmount'': ''noauto'', ''ta:bemount'': ''on''}) }}'
   loop: "{{ jail_list|subelements('persist', skip_missing=True) }}"
 
 - name: Install scripts