diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd index ba98b33..bb1a3c8 100644 --- a/ansible/environments/laptop/host_vars/odofreebsd +++ b/ansible/environments/laptop/host_vars/odofreebsd @@ -44,3 +44,5 @@ jail_list: - name: dagger conf: src: dagger +bhyve_dataset: zroot/freebsd/release/vm +bhyve_list: [] diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index ce84353..716378b 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -27,3 +27,4 @@ - fuse - autofs - exfat + - bhyve diff --git a/ansible/roles/base/files/bemount.bash b/ansible/roles/base/files/bemount.bash new file mode 100644 index 0000000..84054d7 --- /dev/null +++ b/ansible/roles/base/files/bemount.bash @@ -0,0 +1,126 @@ +#!/usr/local/bin/bash +# +# Mount non-boot-environment datasets. +# +# We can't leave datasets outside the boot environment (for example, +# jails or bhyve VMs) as canmount=on because then every boot +# environment's external datasets would all attempt to mount every +# time. To work around this, we mark those datasets as canmount=noauto +# and run this script to mount datasets under the root of our boot +# environment. This script depends heavily on my zfs dataset structure +# so it needs to be improved to be robust enough for different +# layouts. An example of my layout is: +# +## NAME MOUNTPOINT CANMOUNT TA:BEMOUNT +## zroot none off - +## zroot/global /global on - +## zroot/freebsd none on - +## zroot/freebsd/13.1-RELEASE none on - +## zroot/freebsd/13.1-RELEASE/be none on - +## zroot/freebsd/13.1-RELEASE/be/main / noauto - +## zroot/freebsd/13.1-RELEASE/jails none on - +## zroot/freebsd/13.1-RELEASE/jails/foo /jail/foo noauto on +## zroot/freebsd/13.1-RELEASE/jails/bar /jail/bar noauto on +## zroot/freebsd/13.1-RELEASE/jails/baz /jail/baz noauto on +## zroot/freebsd/13.1-RELEASE/vm-bhyve /vm noauto on +## zroot/linux none on - +## zroot/linux/arch none on - +## zroot/linux/arch/be none on - +## zroot/linux/arch/be/main / noauto - +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function main { + local all_zfs_datasets=$(zfs list -Hp -o 'name,mountpoint,canmount,ta:bemount,mounted') + local root_dataset=$(find_root_dataset "$all_zfs_datasets") + local datasets_to_mount=$(find_datasets_to_mount_for_boot_environment "$all_zfs_datasets" "$root_dataset") + if [ -n "$datasets_to_mount" ]; then + mount_datasets "$datasets_to_mount" + fi +} + +function reverse_lines { + sed '1!x;H;1h;$!d;g' +} + +function find_dataset { + local all_zfs_datasets="$1" + local dataset_name="$2" + while read dataset; do + local ds_name=$(awk '{print $1}'<<<"$dataset") + if [ "$ds_name" = "$dataset_name" ]; then + echo "$dataset" + return + fi + done<<<"$all_zfs_datasets" +} + +function find_root_dataset { + local all_zfs_datasets="$1" + while read dataset; do + local ds_name=$(awk '{print $1}'<<<"$dataset") + local ds_mountpoint=$(awk '{print $2}'<<<"$dataset") + # local ds_canmount=$(awk '{print $3}'<<<"$dataset") + # local ds_bemount=$(awk '{print $4}'<<<"$dataset") + local ds_mounted=$(awk '{print $5}'<<<"$dataset") + if [ "$ds_mounted" = "yes" ] && [ "$ds_mountpoint" = "/" ]; then + echo "$ds_name" + return + fi + done<<<"$all_zfs_datasets" +} + +function find_datasets_to_mount_for_boot_environment { + local all_zfs_datasets="$1" + local root_dataset="$2" + # This is a consequence of my layout for zfs datasets. I should + # make this more robust. Perhaps a zfs property like search up + # from dataset mounted at / until you find a dataset with property + # ta:bemountroot="on"? + local be_root_name="${root_dataset%/*/*}" + local be_root_dataset=$(find_dataset "$all_zfs_datasets" "$be_root_name") + + while read dataset; do + local ds_name=$(awk '{print $1}'<<<"$dataset") + # local ds_mountpoint=$(awk '{print $2}'<<<"$dataset") + local ds_canmount=$(awk '{print $3}'<<<"$dataset") + local ds_bemount=$(awk '{print $4}'<<<"$dataset") + local ds_mounted=$(awk '{print $5}'<<<"$dataset") + + case "$ds_name" in + "${be_root_name}/"*) ;; + *) continue ;; + esac + + if [ "$ds_bemount" != "on" ]; then + continue + fi + + if [ "$ds_mounted" != "no" ]; then + continue + fi + + if [ "$ds_canmount" != "noauto" ]; then + continue + fi + echo "$dataset" + + done<<<"$all_zfs_datasets" +} + +function mount_datasets { + local datasets_to_mount=$(reverse_lines<<<"$1") + while read dataset; do + local ds_name=$(awk '{print $1}'<<<"$dataset") + local ds_mountpoint=$(awk '{print $2}'<<<"$dataset") + local ds_canmount=$(awk '{print $3}'<<<"$dataset") + local ds_bemount=$(awk '{print $4}'<<<"$dataset") + local ds_mounted=$(awk '{print $5}'<<<"$dataset") + + mount -v -t zfs "$ds_name" "$ds_mountpoint" + + done<<<"$datasets_to_mount" +} + +main diff --git a/ansible/roles/base/files/bemount_rc.sh b/ansible/roles/base/files/bemount_rc.sh new file mode 100644 index 0000000..b4eaa5c --- /dev/null +++ b/ansible/roles/base/files/bemount_rc.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# REQUIRE: FILESYSTEM kld +# PROVIDE: bemount + +. /etc/rc.subr +name=bemount +rcvar=${name}_enable +start_cmd="${name}_start" +stop_cmd="${name}_stop" +load_rc_config $name + +bemount_start() { + /usr/local/bin/bemount +} + +bemount_stop() { +} + +run_rc_command "$1" diff --git a/ansible/roles/base/tasks/freebsd.yaml b/ansible/roles/base/tasks/freebsd.yaml index 6d3315b..363b2f8 100644 --- a/ansible/roles/base/tasks/freebsd.yaml +++ b/ansible/roles/base/tasks/freebsd.yaml @@ -94,3 +94,31 @@ src: tmpfs fstype: tmpfs opts: rw,mode=777 + +- name: Install scripts + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0755 + owner: root + group: wheel + loop: + - src: bemount.bash + dest: /usr/local/bin/bemount + +- name: Install rc script + copy: + src: "files/{{ item.src }}" + dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}" + owner: root + group: wheel + mode: 0755 + loop: + - src: bemount_rc.sh + dest: bemount + +- name: Enable bemount + community.general.sysrc: + name: bemount_enable + value: "YES" + path: /etc/rc.conf.d/bemount diff --git a/ansible/roles/bhyve/defaults/main.yaml b/ansible/roles/bhyve/defaults/main.yaml new file mode 100644 index 0000000..d7cab58 --- /dev/null +++ b/ansible/roles/bhyve/defaults/main.yaml @@ -0,0 +1,2 @@ +bhyve_mountpoint: "/vm" +bhyve_list: [] diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf new file mode 100644 index 0000000..ef6ff6d --- /dev/null +++ b/ansible/roles/bhyve/files/arch.conf @@ -0,0 +1,22 @@ +# Use UEFI when booting from a disk +loader="uefi" +graphics="yes" +xhci_mouse="yes" +# graphics_listen="0.0.0.0" +graphics_listen="127.0.0.1" +# graphics_listen="10.216.1.1" +graphics_res="1920x1080" +# graphics_wait="yes" +# prestart="" +console="tmux" + +cpu=1 +memory=1024M + +disk0_type="virtio-blk" +disk0_name="disk0" +disk0_dev="sparse-zvol" +virt_random="yes" # virtio-rnd + +# Creates a link to host_bridge1's link3 hook to the vmlink hook on a type socket +bhyve_options="-s 2:0,virtio-net,netgraph,path=host_bridge1:,peerhook=link3" diff --git a/ansible/roles/bhyve/tasks/common.yaml b/ansible/roles/bhyve/tasks/common.yaml new file mode 100644 index 0000000..d7c1735 --- /dev/null +++ b/ansible/roles/bhyve/tasks/common.yaml @@ -0,0 +1,14 @@ +- import_tasks: tasks/freebsd.yaml + when: 'os_flavor == "freebsd"' + +- import_tasks: tasks/linux.yaml + when: 'os_flavor == "linux"' + +- include_tasks: + file: tasks/peruser.yaml + apply: + become: yes + become_user: "{{ initialize_user }}" + loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}" + loop_control: + loop_var: initialize_user diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml new file mode 100644 index 0000000..79f02a5 --- /dev/null +++ b/ansible/roles/bhyve/tasks/freebsd.yaml @@ -0,0 +1,54 @@ +# +# Create a new VM: +# vm iso 'http://mirror.clarkson.edu/archlinux/iso/2022.01.01/archlinux-2022.01.01-x86_64.iso' +# vm create -t arch -s 50G testvm +# vm install testvm 'archlinux-2022.01.01-x86_64.iso' +# +# +- name: Install packages + package: + name: + - vm-bhyve + - tmux # for interactive consoles + - bhyve-firmware # For UEFI + state: present + +- name: Create zfs dataset + zfs: + name: "{{ bhyve_dataset }}" + state: present + extra_zfs_properties: + mountpoint: "{{ bhyve_mountpoint }}" + canmount: "noauto" + "ta:bemount": "on" + +- name: Enable bhyve + community.general.sysrc: + name: "{{ item.name }}" + value: "{{ item.value }}" + path: /etc/rc.conf.d/vm + loop: + - name: vm_enable + value: "YES" + - name: vm_dir + value: "zfs:{{ bhyve_dataset }}" + - name: vm_list + value: "{{ bhyve_list|community.general.json_query('[?enabled==`true`].name')|join(' ') }}" + - name: vm_delay + value: "5" + +- name: init vm-bhyve + command: vm init + args: + creates: "{{ bhyve_mountpoint }}/.templates" + +- name: Install Configuration + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0644 + owner: root + group: wheel + loop: + - src: arch.conf + dest: "{{ bhyve_mountpoint }}/.templates/arch.conf" diff --git a/ansible/roles/bhyve/tasks/linux.yaml b/ansible/roles/bhyve/tasks/linux.yaml new file mode 100644 index 0000000..e1835f0 --- /dev/null +++ b/ansible/roles/bhyve/tasks/linux.yaml @@ -0,0 +1,6 @@ +# - name: Install packages +# pacman: +# name: +# - foo +# state: present +# update_cache: true diff --git a/ansible/roles/bhyve/tasks/main.yaml b/ansible/roles/bhyve/tasks/main.yaml new file mode 100644 index 0000000..639ba76 --- /dev/null +++ b/ansible/roles/bhyve/tasks/main.yaml @@ -0,0 +1,2 @@ +- import_tasks: tasks/common.yaml + when: os_flavor == "freebsd" and bhyve_dataset is defined diff --git a/ansible/roles/bhyve/tasks/peruser.yaml b/ansible/roles/bhyve/tasks/peruser.yaml new file mode 100644 index 0000000..111e886 --- /dev/null +++ b/ansible/roles/bhyve/tasks/peruser.yaml @@ -0,0 +1,29 @@ +- include_role: + name: per_user + +# - name: Create directories +# file: +# name: "{{ account_homedir.stdout }}/{{ item }}" +# state: directory +# mode: 0700 +# owner: "{{ account_name.stdout }}" +# group: "{{ group_name.stdout }}" +# loop: +# - ".config/foo" + +# - name: Copy files +# copy: +# src: "files/{{ item.src }}" +# dest: "{{ account_homedir.stdout }}/{{ item.dest }}" +# mode: 0600 +# owner: "{{ account_name.stdout }}" +# group: "{{ group_name.stdout }}" +# loop: +# - src: foo.conf +# dest: .config/foo/foo.conf + +- import_tasks: tasks/peruser_freebsd.yaml + when: 'os_flavor == "freebsd"' + +- import_tasks: tasks/peruser_linux.yaml + when: 'os_flavor == "linux"' diff --git a/ansible/roles/bhyve/tasks/peruser_freebsd.yaml b/ansible/roles/bhyve/tasks/peruser_freebsd.yaml new file mode 100644 index 0000000..e69de29 diff --git a/ansible/roles/bhyve/tasks/peruser_linux.yaml b/ansible/roles/bhyve/tasks/peruser_linux.yaml new file mode 100644 index 0000000..e69de29 diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf index a672763..f82f37e 100644 --- a/ansible/roles/firewall/files/odofreebsd_pf.conf +++ b/ansible/roles/firewall/files/odofreebsd_pf.conf @@ -41,4 +41,5 @@ pass in on $ext_if proto udp to any port $udp_pass_in pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp pass in on host_uplink0 proto udp from any to any port { 53 51820 } +pass out on host_uplink0 proto tcp from any to any port 8081 pass in on host_uplink1 diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml index e2b7e6e..39368bd 100644 --- a/ansible/roles/jail/tasks/freebsd.yaml +++ b/ansible/roles/jail/tasks/freebsd.yaml @@ -10,7 +10,7 @@ zfs: name: "{{ item.dataset|default(jail_zfs_dataset) }}/jails/{{ item.name }}" state: present - extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({})) }}' + extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({}))|combine({''canmount'': ''noauto'', ''ta:bemount'': ''on''}) }}' loop: "{{ jail_list }}" @@ -27,7 +27,7 @@ zfs: name: "{{ item.0.dataset|default(jail_zfs_dataset) }}/persistent/{{ item.0.name }}/{{ item.1.name }}" state: present - extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({})) }}' + extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({}))|combine({''canmount'': ''noauto'', ''ta:bemount'': ''on''}) }}' loop: "{{ jail_list|subelements('persist', skip_missing=True) }}" - name: Install scripts