From 33d2118c15103ef0b3fa92a535367e3fc0e456c2 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Fri, 28 Oct 2022 21:05:26 -0400
Subject: [PATCH 01/15] Start a jail task which creates the zfs datasets and
 installs a new_jail script.

---
 .../environments/laptop/host_vars/odofreebsd  |  5 +++
 ansible/playbook.yaml                         | 43 ++++++++++---------
 ansible/roles/jail/tasks/common.yaml          | 14 ++++++
 ansible/roles/jail/tasks/freebsd.yaml         | 42 ++++++++++++++++++
 ansible/roles/jail/tasks/linux.yaml           |  6 +++
 ansible/roles/jail/tasks/main.yaml            |  2 +
 ansible/roles/jail/tasks/peruser.yaml         | 29 +++++++++++++
 ansible/roles/jail/tasks/peruser_freebsd.yaml |  0
 ansible/roles/jail/tasks/peruser_linux.yaml   |  0
 ansible/roles/jail/templates/new_jail.bash.j2 | 40 +++++++++++++++++
 10 files changed, 160 insertions(+), 21 deletions(-)
 create mode 100644 ansible/roles/jail/tasks/common.yaml
 create mode 100644 ansible/roles/jail/tasks/freebsd.yaml
 create mode 100644 ansible/roles/jail/tasks/linux.yaml
 create mode 100644 ansible/roles/jail/tasks/main.yaml
 create mode 100644 ansible/roles/jail/tasks/peruser.yaml
 create mode 100644 ansible/roles/jail/tasks/peruser_freebsd.yaml
 create mode 100644 ansible/roles/jail/tasks/peruser_linux.yaml
 create mode 100644 ansible/roles/jail/templates/new_jail.bash.j2

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 294c60b..fd2fc3a 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -35,3 +35,8 @@ users:
     gitconfig: "gitconfig_home"
 # devfs_rules: "odo_devfs.rules"
 # devfs_system_ruleset: "localrules"
+# jail_conf: "jail.conf"
+jail_zfs_dataset: zroot/freebsd/release/jails
+jail_zfs_dataset_mountpoint: /jail/main
+jail_list:
+  - name: cloak
diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index d6bbd97..6e7fb63 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -2,24 +2,25 @@
   vars:
     ansible_become: True
   roles:
-    - sudo
-    - users
-    - package_manager
-    - zrepl
-    - zsh
-    - network
-    - sshd
-    - base
-    - firewall
-    - cpu
-    - ntp
-    - build
-    - graphics
-    - gpg
-    - fonts
-    - alacritty
-    - sway
-    - emacs
-    - firefox
-    - devfs
-    - ssh_client
+    # - sudo
+    # - users
+    # - package_manager
+    # - zrepl
+    # - zsh
+    # - network
+    # - sshd
+    # - base
+    # #    - firewall
+    # - cpu
+    # - ntp
+    # - build
+    # - graphics
+    # - gpg
+    # - fonts
+    # - alacritty
+    # - sway
+    # - emacs
+    # - firefox
+    # - devfs
+    # - ssh_client
+    - jail
diff --git a/ansible/roles/jail/tasks/common.yaml b/ansible/roles/jail/tasks/common.yaml
new file mode 100644
index 0000000..d7c1735
--- /dev/null
+++ b/ansible/roles/jail/tasks/common.yaml
@@ -0,0 +1,14 @@
+- import_tasks: tasks/freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/linux.yaml
+  when: 'os_flavor == "linux"'
+
+- include_tasks:
+    file: tasks/peruser.yaml
+    apply:
+      become: yes
+      become_user: "{{ initialize_user }}"
+  loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}"
+  loop_control:
+    loop_var: initialize_user
diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
new file mode 100644
index 0000000..92d5b5e
--- /dev/null
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -0,0 +1,42 @@
+- name: Create common zfs datasets
+  zfs:
+    name: "{{ item }}"
+    state: present
+    extra_zfs_properties:
+      mountpoint: "none"
+  loop: "{{ ((jail_list | community.general.json_query('[*].dataset')) + [jail_zfs_dataset]) | product(['', '/persistent', '/jails']) | map('join', '') }}"
+
+- name: Create jail zfs datasets
+  zfs:
+    name: "{{ item.dataset|default(jail_zfs_dataset) }}/jails/{{ item.name }}"
+    state: present
+    extra_zfs_properties: '{{ {''mountpoint'': item.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.name}|combine(item.properties|default({})) }}'
+
+  loop: "{{ jail_list }}"
+
+- name: Create persistent jail zfs datasets
+  zfs:
+    name: "{{ item.dataset|default(jail_zfs_dataset) }}/persistent/{{ item.name }}"
+    state: present
+    extra_zfs_properties:
+      mountpoint: "none"
+  when: item.persist|default([])|length > 0
+  loop: "{{ jail_list }}"
+
+- name: Create jail specific zfs datasets
+  zfs:
+    name: "{{ item.0.dataset|default(jail_zfs_dataset) }}/persistent/{{ item.0.name }}/{{ item.1.name }}"
+    state: present
+    extra_zfs_properties: '{{ {''mountpoint'': item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) + "/jails/" + item.0.name + item.1.mount }|combine(item.1.properties|default({})) }}'
+  loop: "{{ jail_list|subelements('persist', skip_missing=True) }}"
+
+- name: Install scripts
+  template:
+    src: "templates/{{ item.src }}.j2"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: new_jail.bash
+      dest: /usr/local/bin/new_jail
diff --git a/ansible/roles/jail/tasks/linux.yaml b/ansible/roles/jail/tasks/linux.yaml
new file mode 100644
index 0000000..e1835f0
--- /dev/null
+++ b/ansible/roles/jail/tasks/linux.yaml
@@ -0,0 +1,6 @@
+# - name: Install packages
+#   pacman:
+#     name:
+#       - foo
+#     state: present
+#     update_cache: true
diff --git a/ansible/roles/jail/tasks/main.yaml b/ansible/roles/jail/tasks/main.yaml
new file mode 100644
index 0000000..b7fbdd3
--- /dev/null
+++ b/ansible/roles/jail/tasks/main.yaml
@@ -0,0 +1,2 @@
+- import_tasks: tasks/common.yaml
+  when: jail_zfs_dataset is defined and jail_zfs_dataset_mountpoint is defined
diff --git a/ansible/roles/jail/tasks/peruser.yaml b/ansible/roles/jail/tasks/peruser.yaml
new file mode 100644
index 0000000..111e886
--- /dev/null
+++ b/ansible/roles/jail/tasks/peruser.yaml
@@ -0,0 +1,29 @@
+- include_role:
+    name: per_user
+
+# - name: Create directories
+#   file:
+#     name: "{{ account_homedir.stdout }}/{{ item }}"
+#     state: directory
+#     mode: 0700
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - ".config/foo"
+
+# - name: Copy files
+#   copy:
+#     src: "files/{{ item.src }}"
+#     dest: "{{ account_homedir.stdout }}/{{ item.dest }}"
+#     mode: 0600
+#     owner: "{{ account_name.stdout }}"
+#     group: "{{ group_name.stdout }}"
+#   loop:
+#     - src: foo.conf
+#       dest: .config/foo/foo.conf
+
+- import_tasks: tasks/peruser_freebsd.yaml
+  when: 'os_flavor == "freebsd"'
+
+- import_tasks: tasks/peruser_linux.yaml
+  when: 'os_flavor == "linux"'
diff --git a/ansible/roles/jail/tasks/peruser_freebsd.yaml b/ansible/roles/jail/tasks/peruser_freebsd.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/ansible/roles/jail/tasks/peruser_linux.yaml b/ansible/roles/jail/tasks/peruser_linux.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/ansible/roles/jail/templates/new_jail.bash.j2 b/ansible/roles/jail/templates/new_jail.bash.j2
new file mode 100644
index 0000000..619fa38
--- /dev/null
+++ b/ansible/roles/jail/templates/new_jail.bash.j2
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Create a new jail
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: ${JAIL_MOUNTPOINT:="{{ jail_zfs_dataset_mountpoint }}/jails"}
+
+function die {
+    echo >&2 "$@"
+    exit 1
+}
+
+[ "$#" -eq 2 ] || die "1 argument required, $# provided"
+
+JAIL_NAME="$2"
+export DESTDIR="${JAIL_MOUNTPOINT}/$JAIL_NAME"
+
+function by_src {
+    cd /usr/src
+    make -j 16 buildworld
+    make installworld DESTDIR=$DESTDIR
+    make distribution DESTDIR=$DESTDIR
+}
+
+function by_bin {
+    DESTRELEASE=13.1-RELEASE
+    DESTARCH=`uname -m`
+    SOURCEURL=http://ftp.freebsd.org/pub/FreeBSD/releases/$DESTARCH/$DESTRELEASE/
+    for component in base ports; do fetch $SOURCEURL/$component.txz -o - | tar -xf - -C "$DESTDIR" ; done
+}
+
+if [ "$1" = "src" ]; then
+    by_src
+elif [ "$1" = "bin" ]; then
+    by_bin
+else
+    die "First argument must be either 'src' or 'bin', got $1"
+fi

From 46f182803e3761c68a61816f8ec9166c878375b5 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Fri, 28 Oct 2022 21:24:06 -0400
Subject: [PATCH 02/15] Add enabling startup jails.

---
 ansible/roles/jail/tasks/freebsd.yaml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index 92d5b5e..15f6989 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -40,3 +40,23 @@
   loop:
     - src: new_jail.bash
       dest: /usr/local/bin/new_jail
+
+- name: Enable Jails
+  community.general.sysrc:
+    name: jail_enable
+    value: "YES"
+    path: /etc/rc.conf.d/jail
+  when: jail_list|community.general.json_query('[?enabled==`true`]')|length > 0
+
+- name: Set enabled jail list
+  community.general.sysrc:
+    name: jail_list
+    value: "{{ jail_list|community.general.json_query('[?enabled==`true`].name')|join(' ') }}"
+    path: /etc/rc.conf.d/jail
+  when: jail_list|community.general.json_query('[?enabled==`true`]')|length > 0
+
+- name: Disable Jails
+  file:
+    path: /etc/rc.conf.d/jail
+    state: absent
+  when: jail_list|community.general.json_query('[?enabled==`true`]')|length == 0

From a2945ff069175723fb743e32e7091f163262491c Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Fri, 28 Oct 2022 21:26:15 -0400
Subject: [PATCH 03/15] Add copying of fstab and persistent files for jails.

---
 ansible/roles/jail/tasks/freebsd.yaml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index 15f6989..97db82d 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -41,6 +41,25 @@
     - src: new_jail.bash
       dest: /usr/local/bin/new_jail
 
+- name: Install config files
+  copy:
+    src: "files/{{ item.fstab }}"
+    dest: '{{ item.fstab_dest|default("/etc/fstab." + item.name) }}'
+    mode: 0644
+    owner: root
+    group: wheel
+  when: item.fstab is defined
+  loop: "{{ jail_list }}"
+
+- name: Install persistent files
+  copy:
+    src: "files/{{ item.1.src }}"
+    dest: "{{ item.0.dataset_mountpoint|default(jail_zfs_dataset_mountpoint) }}/jails/{{ item.0.name }}{{ item.1.dest }}"
+    mode: '{{ item.1.mode|default("0644") }}'
+    owner: root
+    group: wheel
+  loop: "{{ jail_list|subelements('files', skip_missing=True) }}"
+
 - name: Enable Jails
   community.general.sysrc:
     name: jail_enable

From 7db98bc9d14af89661e16431c159d81177766ad0 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Fri, 28 Oct 2022 21:51:08 -0400
Subject: [PATCH 04/15] Add support for jail.conf.d files.

---
 ansible/environments/laptop/host_vars/odofreebsd |  2 ++
 ansible/roles/jail/files/jails/cloak.conf        |  0
 ansible/roles/jail/tasks/freebsd.yaml            | 11 +++++++++++
 3 files changed, 13 insertions(+)
 create mode 100644 ansible/roles/jail/files/jails/cloak.conf

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index fd2fc3a..0832903 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -40,3 +40,5 @@ jail_zfs_dataset: zroot/freebsd/release/jails
 jail_zfs_dataset_mountpoint: /jail/main
 jail_list:
   - name: cloak
+    conf:
+      src: cloak
diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
new file mode 100644
index 0000000..e69de29
diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index 97db82d..db99c14 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -60,6 +60,16 @@
     group: wheel
   loop: "{{ jail_list|subelements('files', skip_missing=True) }}"
 
+- name: Install jail.conf files
+  when: item.conf.src is defined
+  copy:
+    src: "files/jails/{{ item.conf.src }}.conf"
+    dest: "/etc/jail.conf.d/{{ item.conf.dest|default(item.conf.src) }}.conf"
+    mode: "0644"
+    owner: root
+    group: wheel
+  loop: "{{ jail_list }}"
+
 - name: Enable Jails
   community.general.sysrc:
     name: jail_enable
@@ -79,3 +89,4 @@
     path: /etc/rc.conf.d/jail
     state: absent
   when: jail_list|community.general.json_query('[?enabled==`true`]')|length == 0
+

From 156261f082fa2449ff0ddbcdf2e51ac89a56b394 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Fri, 28 Oct 2022 22:24:32 -0400
Subject: [PATCH 05/15] A very basic jail config for cloak.

---
 ansible/roles/jail/files/jails/cloak.conf | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
index e69de29..7a5bf1e 100644
--- a/ansible/roles/jail/files/jails/cloak.conf
+++ b/ansible/roles/jail/files/jails/cloak.conf
@@ -0,0 +1,8 @@
+cloak {
+    path = "/jail/main/jails/cloak";
+    ip4.addr += "lo0|127.0.0.2";
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}

From c6ed886f4c16e13810eef338b31fb4ed82438662 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 15:55:34 -0400
Subject: [PATCH 06/15] Add default jail list.

---
 ansible/roles/jail/defaults/main.yaml     | 1 +
 ansible/roles/jail/files/jails/cloak.conf | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 ansible/roles/jail/defaults/main.yaml

diff --git a/ansible/roles/jail/defaults/main.yaml b/ansible/roles/jail/defaults/main.yaml
new file mode 100644
index 0000000..7c6d77d
--- /dev/null
+++ b/ansible/roles/jail/defaults/main.yaml
@@ -0,0 +1 @@
+jail_list: []
diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
index 7a5bf1e..db56473 100644
--- a/ansible/roles/jail/files/jails/cloak.conf
+++ b/ansible/roles/jail/files/jails/cloak.conf
@@ -1,6 +1,7 @@
 cloak {
     path = "/jail/main/jails/cloak";
-    ip4.addr += "lo0|127.0.0.2";
+    vnet;
+    vnet.interface += "nat_link2";
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";

From 4e29c2e3d8f8c1c3e16dbc8954f19d406fc18436 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 17:45:06 -0400
Subject: [PATCH 07/15] Starting an rc file to set up netgraph.

---
 .../environments/laptop/host_vars/odofreebsd  |  1 +
 ansible/roles/jail/files/setup_netgraph       | 23 +++++++++++
 ansible/roles/jail/files/setup_netgraph_odo   | 41 +++++++++++++++++++
 ansible/roles/jail/tasks/freebsd.yaml         | 35 ++++++++++++++++
 4 files changed, 100 insertions(+)
 create mode 100644 ansible/roles/jail/files/setup_netgraph
 create mode 100644 ansible/roles/jail/files/setup_netgraph_odo

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 0832903..03ba232 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -10,6 +10,7 @@ pflog_conf:
 network_rc: "odofreebsd_network.conf"
 rc_conf: "odofreebsd_rc.conf"
 loader_conf: "odofreebsd_loader.conf"
+netgraph_config: "setup_netgraph_odo"
 graphics_driver: "intel"
 cputype: "intel"
 cpu_opt: tigerlake
diff --git a/ansible/roles/jail/files/setup_netgraph b/ansible/roles/jail/files/setup_netgraph
new file mode 100644
index 0000000..e6bc224
--- /dev/null
+++ b/ansible/roles/jail/files/setup_netgraph
@@ -0,0 +1,23 @@
+#!/bin/sh
+# /usr/local/etc/rc.d/setup_netgraph
+#
+# REQUIRE: FILESYSTEM kld
+# PROVIDE: setup_netgraph
+# BEFORE: netif
+
+. /etc/rc.subr
+name=setup_netgraph
+rcvar=${name}_enable
+start_cmd="${name}_start"
+stop_cmd="${name}_stop"
+load_rc_config $name
+
+setup_netgraph_start() {
+    /usr/local/bin/setup_netgraph start
+}
+
+setup_netgraph_stop() {
+    /usr/local/bin/setup_netgraph stop
+}
+
+run_rc_command "$1"
diff --git a/ansible/roles/jail/files/setup_netgraph_odo b/ansible/roles/jail/files/setup_netgraph_odo
new file mode 100644
index 0000000..3241a27
--- /dev/null
+++ b/ansible/roles/jail/files/setup_netgraph_odo
@@ -0,0 +1,41 @@
+#!/usr/local/bin/bash
+
+cleanup() {
+    ngctl shutdown nat_link2:
+    ngctl shutdown nat_uplink0:
+    ngctl shutdown jail_nat_wg0:
+}
+
+setup_netgraph_start() {
+    cleanup
+
+    ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook nat_uplink0
+EOF
+
+    ngctl -d -f - <<EOF
+mkpeer nat_uplink0: bridge ether link0
+name nat_uplink0:ether jail_nat_wg0
+EOF
+
+    ngctl -d -f - <<EOF
+mkpeer jail_nat_wg0: eiface link2 ether
+name jail_nat_wg0:link2 nat_link2
+EOF
+
+    ifconfig $(ngctl msg 'nat_uplink0:' getifname | grep Args | cut -d '"' -f 2) name nat_uplink0 10.10.11.1/24 up
+    ifconfig $(ngctl msg 'jail_nat_wg0:link2' getifname | grep Args | cut -d '"' -f 2) name nat_link2
+}
+
+setup_netgraph_stop() {
+    cleanup
+}
+
+if [ "$1" = "start" ]; then
+    setup_netgraph_start
+elif [ "$1" = "stop" ]; then
+    setup_netgraph_stop
+else
+    >&2 echo "Unrecognized command"
+fi
diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index db99c14..2ab0242 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -90,3 +90,38 @@
     state: absent
   when: jail_list|community.general.json_query('[?enabled==`true`]')|length == 0
 
+- name: Install rc script
+  when: netgraph_config is defined
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "/usr/local/etc/rc.d/{{ item.dest|default(item.src) }}"
+    owner: root
+    group: wheel
+    mode: 0755
+  loop:
+    - src: setup_netgraph
+
+- name: Install scripts
+  when: netgraph_config is defined
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: "{{ netgraph_config }}"
+      dest: /usr/local/bin/setup_netgraph
+
+- name: Enable setup_netgraph
+  when: netgraph_config is defined
+  community.general.sysrc:
+    name: setup_netgraph_enable
+    value: "YES"
+    path: /etc/rc.conf.d/setup_netgraph
+
+- name: Disable setup_netgraph
+  when: netgraph_config is not defined
+  file:
+    path: /etc/rc.conf.d/setup_netgraph
+    state: absent

From 2e893733a87fe7d3935f7daa6037e7d92a6aef68 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 19:49:25 -0400
Subject: [PATCH 08/15] NAT working but not fail-safe.

When the firewall is down, packets still go out wlan0 but with untranslated source ips.
---
 ansible/playbook.yaml                           | 2 +-
 ansible/roles/firewall/files/odofreebsd_pf.conf | 7 +++++--
 ansible/roles/jail/tasks/freebsd.yaml           | 9 +++++++++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index 6e7fb63..34a0a74 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -10,7 +10,7 @@
     # - network
     # - sshd
     # - base
-    # #    - firewall
+    - firewall
     # - cpu
     # - ntp
     # - build
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index fa5f23c..1b942d0 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -1,7 +1,8 @@
 ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
+jail_net_v4 = "10.10.11.0/24"
 
 dhcp = "{ bootpc, bootps }"
-# allow = "{ }"
+allow = "{ nat_uplink0 }"
 
 tcp_pass_in = "{ 22 }"
 udp_pass_in = "{ 53 51820 }"
@@ -12,9 +13,11 @@ udp_pass_in = "{ 53 51820 }"
 set skip on lo
 
 # redirections
+nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
 
 # filtering
 block log all
+pass quick on $ext_if tagged ALLOWED
 pass out on $ext_if
 
 # We pass on the interfaces listed in allow rather than skipping on
@@ -22,7 +25,7 @@ pass out on $ext_if
 # `service pf reload` but interfaces that we `skip` will not update (I
 # forget if its from adding, removing, or both. TODO: test to figure
 # it out)
-# pass quick on $allow
+pass quick on $allow
 
 pass on $ext_if proto icmp all
 pass on $ext_if proto icmp6 all
diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml
index 2ab0242..e2b7e6e 100644
--- a/ansible/roles/jail/tasks/freebsd.yaml
+++ b/ansible/roles/jail/tasks/freebsd.yaml
@@ -125,3 +125,12 @@
   file:
     path: /etc/rc.conf.d/setup_netgraph
     state: absent
+
+- name: Enable gateway
+  community.general.sysrc:
+    name: "{{ item }}"
+    value: "YES"
+    path: /etc/rc.conf.d/routing
+  loop:
+    - gateway_enable
+    - ipv6_gateway_enable

From 4de74765af9f8c1100ce27839be61e26f4db8316 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 20:11:25 -0400
Subject: [PATCH 09/15] Switch to a different ip address range.

---
 ansible/roles/firewall/files/odofreebsd_pf.conf | 2 +-
 ansible/roles/jail/files/setup_netgraph_odo     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 1b942d0..33851d9 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -1,5 +1,5 @@
 ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
-jail_net_v4 = "10.10.11.0/24"
+jail_net_v4 = "10.193.223.0/24"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ nat_uplink0 }"
diff --git a/ansible/roles/jail/files/setup_netgraph_odo b/ansible/roles/jail/files/setup_netgraph_odo
index 3241a27..86eefb0 100644
--- a/ansible/roles/jail/files/setup_netgraph_odo
+++ b/ansible/roles/jail/files/setup_netgraph_odo
@@ -24,7 +24,7 @@ mkpeer jail_nat_wg0: eiface link2 ether
 name jail_nat_wg0:link2 nat_link2
 EOF
 
-    ifconfig $(ngctl msg 'nat_uplink0:' getifname | grep Args | cut -d '"' -f 2) name nat_uplink0 10.10.11.1/24 up
+    ifconfig $(ngctl msg 'nat_uplink0:' getifname | grep Args | cut -d '"' -f 2) name nat_uplink0 10.193.223.1/24 up
     ifconfig $(ngctl msg 'jail_nat_wg0:link2' getifname | grep Args | cut -d '"' -f 2) name nat_link2
 }
 

From 464d873b316ed58c24faad3dc2a74a308dd7163d Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 20:36:13 -0400
Subject: [PATCH 10/15] DNS forwarding working.

---
 ansible/roles/firewall/files/odofreebsd_pf.conf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 33851d9..56b4db7 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -15,6 +15,8 @@ set skip on lo
 # redirections
 nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
 
+rdr on nat_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+
 # filtering
 block log all
 pass quick on $ext_if tagged ALLOWED

From 016ed084407ffc0c264a136fb8aa351fb906698d Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 21:29:00 -0400
Subject: [PATCH 11/15] Set up the cloak jail to run wireguard.

---
 ansible/environments/laptop/host_vars/odofreebsd |  2 +-
 ansible/roles/devfs/files/odo_devfs.rules        | 10 ++++++++++
 ansible/roles/firewall/files/odofreebsd_pf.conf  |  6 ++++--
 ansible/roles/jail/files/jails/cloak.conf        |  2 ++
 ansible/roles/jail/meta/main.yaml                |  2 ++
 5 files changed, 19 insertions(+), 3 deletions(-)
 create mode 100644 ansible/roles/jail/meta/main.yaml

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 03ba232..13a3cba 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -34,7 +34,7 @@ users:
       - backup_fido
       - homeassistant
     gitconfig: "gitconfig_home"
-# devfs_rules: "odo_devfs.rules"
+devfs_rules: "odo_devfs.rules"
 # devfs_system_ruleset: "localrules"
 # jail_conf: "jail.conf"
 jail_zfs_dataset: zroot/freebsd/release/jails
diff --git a/ansible/roles/devfs/files/odo_devfs.rules b/ansible/roles/devfs/files/odo_devfs.rules
index 273e8a1..6797aad 100644
--- a/ansible/roles/devfs/files/odo_devfs.rules
+++ b/ansible/roles/devfs/files/odo_devfs.rules
@@ -1,3 +1,13 @@
 [localrules=10]
 add path 'input/*' mode 0660 group video
 add path 'usb/*' mode 0660 group usb
+
+[tajailwg=13]
+add include $devfsrules_hide_all
+add include $devfsrules_unhide_basic
+add include $devfsrules_unhide_login
+add path 'bpf*' unhide
+add path pf unhide
+add path pflog unhide
+add path pfsynv unhide
+add path 'tun*' unhide
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 56b4db7..4d6f688 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -2,7 +2,7 @@ ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
 jail_net_v4 = "10.193.223.0/24"
 
 dhcp = "{ bootpc, bootps }"
-allow = "{ nat_uplink0 }"
+# allow = "{ nat_uplink0 }"
 
 tcp_pass_in = "{ 22 }"
 udp_pass_in = "{ 53 51820 }"
@@ -27,7 +27,7 @@ pass out on $ext_if
 # `service pf reload` but interfaces that we `skip` will not update (I
 # forget if its from adding, removing, or both. TODO: test to figure
 # it out)
-pass quick on $allow
+# pass quick on $allow
 
 pass on $ext_if proto icmp all
 pass on $ext_if proto icmp6 all
@@ -36,3 +36,5 @@ pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
 
 pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
+
+pass in on nat_uplink0 proto udp from any to any port { 53 51820 }
diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
index db56473..7026d39 100644
--- a/ansible/roles/jail/files/jails/cloak.conf
+++ b/ansible/roles/jail/files/jails/cloak.conf
@@ -2,6 +2,8 @@ cloak {
     path = "/jail/main/jails/cloak";
     vnet;
     vnet.interface += "nat_link2";
+    devfs_ruleset = 13;
+    mount.devfs; # To expose tun device
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";
diff --git a/ansible/roles/jail/meta/main.yaml b/ansible/roles/jail/meta/main.yaml
new file mode 100644
index 0000000..b5b170a
--- /dev/null
+++ b/ansible/roles/jail/meta/main.yaml
@@ -0,0 +1,2 @@
+dependencies:
+  - devfs # To expose /dev entries like tun for wireguard

From 9168cc51cf163562eee460d370662633b227c3c3 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 21:47:32 -0400
Subject: [PATCH 12/15] Rename the interfaces to make the separate levels more
 clear.

---
 ansible/roles/base/tasks/freebsd.yaml         |  1 +
 .../roles/firewall/files/odofreebsd_pf.conf   |  6 ++---
 ansible/roles/jail/files/jails/cloak.conf     |  2 +-
 ansible/roles/jail/files/setup_netgraph_odo   | 22 ++++++++++---------
 4 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/ansible/roles/base/tasks/freebsd.yaml b/ansible/roles/base/tasks/freebsd.yaml
index 88a6e65..0b3dbc8 100644
--- a/ansible/roles/base/tasks/freebsd.yaml
+++ b/ansible/roles/base/tasks/freebsd.yaml
@@ -2,6 +2,7 @@
   package:
     name:
       - pstree
+      - gsed
     state: present
 
 - name: See if the alacritty termcap has been added
diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf
index 4d6f688..f7649ff 100644
--- a/ansible/roles/firewall/files/odofreebsd_pf.conf
+++ b/ansible/roles/firewall/files/odofreebsd_pf.conf
@@ -2,7 +2,7 @@ ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
 jail_net_v4 = "10.193.223.0/24"
 
 dhcp = "{ bootpc, bootps }"
-# allow = "{ nat_uplink0 }"
+# allow = "{ host_uplink0 }"
 
 tcp_pass_in = "{ 22 }"
 udp_pass_in = "{ 53 51820 }"
@@ -15,7 +15,7 @@ set skip on lo
 # redirections
 nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
 
-rdr on nat_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+rdr on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
 
 # filtering
 block log all
@@ -37,4 +37,4 @@ pass in on $ext_if proto udp to any port $udp_pass_in
 
 pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
 
-pass in on nat_uplink0 proto udp from any to any port { 53 51820 }
+pass in on host_uplink0 proto udp from any to any port { 53 51820 }
diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
index 7026d39..9c413c3 100644
--- a/ansible/roles/jail/files/jails/cloak.conf
+++ b/ansible/roles/jail/files/jails/cloak.conf
@@ -1,7 +1,7 @@
 cloak {
     path = "/jail/main/jails/cloak";
     vnet;
-    vnet.interface += "nat_link2";
+    vnet.interface += "host_link2";
     devfs_ruleset = 13;
     mount.devfs; # To expose tun device
 
diff --git a/ansible/roles/jail/files/setup_netgraph_odo b/ansible/roles/jail/files/setup_netgraph_odo
index 86eefb0..10a6dfe 100644
--- a/ansible/roles/jail/files/setup_netgraph_odo
+++ b/ansible/roles/jail/files/setup_netgraph_odo
@@ -1,9 +1,9 @@
 #!/usr/local/bin/bash
 
 cleanup() {
-    ngctl shutdown nat_link2:
-    ngctl shutdown nat_uplink0:
-    ngctl shutdown jail_nat_wg0:
+    ngctl shutdown host_link2:
+    ngctl shutdown host_uplink0:
+    ngctl shutdown host_bridge0:
 }
 
 setup_netgraph_start() {
@@ -11,21 +11,23 @@ setup_netgraph_start() {
 
     ngctl -d -f - <<EOF
 mkpeer . eiface hook ether
-name .:hook nat_uplink0
+name .:hook host_uplink0
 EOF
 
     ngctl -d -f - <<EOF
-mkpeer nat_uplink0: bridge ether link0
-name nat_uplink0:ether jail_nat_wg0
+mkpeer host_uplink0: bridge ether link0
+name host_uplink0:ether host_bridge0
 EOF
 
     ngctl -d -f - <<EOF
-mkpeer jail_nat_wg0: eiface link2 ether
-name jail_nat_wg0:link2 nat_link2
+mkpeer host_bridge0: eiface link2 ether
+name host_bridge0:link2 host_link2
 EOF
 
-    ifconfig $(ngctl msg 'nat_uplink0:' getifname | grep Args | cut -d '"' -f 2) name nat_uplink0 10.193.223.1/24 up
-    ifconfig $(ngctl msg 'jail_nat_wg0:link2' getifname | grep Args | cut -d '"' -f 2) name nat_link2
+    ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
+    ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
+
+    # Create internal bridge for jails that are forced through wireguard
 }
 
 setup_netgraph_stop() {

From 68069a17b4b8e34abf445abfa905ac88bd64beeb Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 22:44:35 -0400
Subject: [PATCH 13/15] Add a bridge for jails behind wireguard.

---
 ansible/roles/jail/files/jails/cloak.conf   |  1 +
 ansible/roles/jail/files/setup_netgraph_odo | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf
index 9c413c3..181974f 100644
--- a/ansible/roles/jail/files/jails/cloak.conf
+++ b/ansible/roles/jail/files/jails/cloak.conf
@@ -2,6 +2,7 @@ cloak {
     path = "/jail/main/jails/cloak";
     vnet;
     vnet.interface += "host_link2";
+    vnet.interface += "wg_uplink0";
     devfs_ruleset = 13;
     mount.devfs; # To expose tun device
 
diff --git a/ansible/roles/jail/files/setup_netgraph_odo b/ansible/roles/jail/files/setup_netgraph_odo
index 10a6dfe..7ed26c9 100644
--- a/ansible/roles/jail/files/setup_netgraph_odo
+++ b/ansible/roles/jail/files/setup_netgraph_odo
@@ -4,6 +4,9 @@ cleanup() {
     ngctl shutdown host_link2:
     ngctl shutdown host_uplink0:
     ngctl shutdown host_bridge0:
+    ngctl shutdown wg_link2:
+    ngctl shutdown wg_uplink0:
+    ngctl shutdown wg_bridge0:
 }
 
 setup_netgraph_start() {
@@ -28,6 +31,23 @@ EOF
     ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
 
     # Create internal bridge for jails that are forced through wireguard
+    ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook wg_uplink0
+EOF
+
+    ngctl -d -f - <<EOF
+mkpeer wg_uplink0: bridge ether link0
+name wg_uplink0:ether wg_bridge0
+EOF
+
+    ngctl -d -f - <<EOF
+mkpeer wg_bridge0: eiface link2 ether
+name wg_bridge0:link2 wg_link2
+EOF
+
+    ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
+    ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
 }
 
 setup_netgraph_stop() {

From 93eb8e331a3be9079fcd5ddfe167ec5fd688fa62 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 23:50:07 -0400
Subject: [PATCH 14/15] Add a dagger jail.

---
 ansible/environments/laptop/host_vars/odofreebsd | 3 +++
 ansible/roles/jail/files/jails/dagger.conf       | 9 +++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 ansible/roles/jail/files/jails/dagger.conf

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 13a3cba..40531f2 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -43,3 +43,6 @@ jail_list:
   - name: cloak
     conf:
       src: cloak
+  - name: dagger
+    conf:
+      src: dagger
diff --git a/ansible/roles/jail/files/jails/dagger.conf b/ansible/roles/jail/files/jails/dagger.conf
new file mode 100644
index 0000000..cab230e
--- /dev/null
+++ b/ansible/roles/jail/files/jails/dagger.conf
@@ -0,0 +1,9 @@
+dagger {
+    path = "/jail/main/jails/dagger";
+    vnet;
+    vnet.interface += "wg_link2";
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}

From adf825839d3e2901a2ca386715163ee4d7caee4a Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Sat, 29 Oct 2022 23:52:55 -0400
Subject: [PATCH 15/15] cleanup

---
 .../environments/laptop/host_vars/odofreebsd  |  2 -
 ansible/playbook.yaml                         | 40 +++++++++----------
 2 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd
index 40531f2..ba98b33 100644
--- a/ansible/environments/laptop/host_vars/odofreebsd
+++ b/ansible/environments/laptop/host_vars/odofreebsd
@@ -35,8 +35,6 @@ users:
       - homeassistant
     gitconfig: "gitconfig_home"
 devfs_rules: "odo_devfs.rules"
-# devfs_system_ruleset: "localrules"
-# jail_conf: "jail.conf"
 jail_zfs_dataset: zroot/freebsd/release/jails
 jail_zfs_dataset_mountpoint: /jail/main
 jail_list:
diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml
index 34a0a74..fb1182f 100644
--- a/ansible/playbook.yaml
+++ b/ansible/playbook.yaml
@@ -2,25 +2,25 @@
   vars:
     ansible_become: True
   roles:
-    # - sudo
-    # - users
-    # - package_manager
-    # - zrepl
-    # - zsh
-    # - network
-    # - sshd
-    # - base
+    - sudo
+    - users
+    - package_manager
+    - zrepl
+    - zsh
+    - network
+    - sshd
+    - base
     - firewall
-    # - cpu
-    # - ntp
-    # - build
-    # - graphics
-    # - gpg
-    # - fonts
-    # - alacritty
-    # - sway
-    # - emacs
-    # - firefox
-    # - devfs
-    # - ssh_client
+    - cpu
+    - ntp
+    - build
+    - graphics
+    - gpg
+    - fonts
+    - alacritty
+    - sway
+    - emacs
+    - firefox
+    - devfs
+    - ssh_client
     - jail