diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd index 6a44b37..f402874 100644 --- a/ansible/environments/laptop/host_vars/odofreebsd +++ b/ansible/environments/laptop/host_vars/odofreebsd @@ -10,7 +10,6 @@ pflog_conf: network_rc: "odofreebsd_network.conf" rc_conf: "odofreebsd_rc.conf" loader_conf: "odofreebsd_loader.conf" -netgraph_config: "setup_netgraph_odo" install_graphics: true graphics_driver: "intel" cputype: "intel" diff --git a/ansible/roles/base/tasks/freebsd.yaml b/ansible/roles/base/tasks/freebsd.yaml index 0be7959..5edd441 100644 --- a/ansible/roles/base/tasks/freebsd.yaml +++ b/ansible/roles/base/tasks/freebsd.yaml @@ -4,6 +4,7 @@ - pstree - gsed - gmake + - rust-coreutils state: present - name: See if the alacritty termcap has been added diff --git a/ansible/roles/base/tasks/linux.yaml b/ansible/roles/base/tasks/linux.yaml index 943fadd..67b3554 100644 --- a/ansible/roles/base/tasks/linux.yaml +++ b/ansible/roles/base/tasks/linux.yaml @@ -6,6 +6,7 @@ - linux-firmware - bind # dig - man-db + - uutils-coreutils state: present - name: Start pkgfile update service diff --git a/ansible/roles/bhyve/files/arch.conf b/ansible/roles/bhyve/files/arch.conf index 447049e..4127080 100644 --- a/ansible/roles/bhyve/files/arch.conf +++ b/ansible/roles/bhyve/files/arch.conf @@ -19,7 +19,7 @@ disk0_dev="sparse-zvol" virt_random="yes" # virtio-rnd # Creates a link to host_bridge1's link3 hook to the vmlink hook on a type socket -bhyve_options="-s 2:0,virtio-net,netgraph,path=host_bridge1:,peerhook=link3" +bhyve_options="-s 2:0,virtio-net,netgraph,path=bridge_jail_nat:,peerhook=link90" # Share a host directory to the guest via 9pfs. # diff --git a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash new file mode 100644 index 0000000..23db06c --- /dev/null +++ b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash @@ -0,0 +1,142 @@ +#!/usr/local/bin/bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function main { + if [ "$1" = "create-disk" ]; then + shift 1 + create_disk "${@}" + elif [ "$1" = "start" ]; then + shift 1 + start_vm "${@}" + else + >&2 echo "Unrecognized command" + exit 1 + fi +} + +function create_disk { + zfs_path="$1" + mount_path="$2" + gigabytes="$3" + zfs create -o "mountpoint=$mount_path" "$zfs_path" + cp /usr/local/share/edk2-bhyve/BHYVE_UEFI_VARS.fd "${mount_path}/" + tee "${mount_path}/settings" <&2 echo "No available links on bridge $bridge_name") + exit 1 + fi + done +} + +function assert_bridge { + host_interface_name="$1" + bridge_name="$2" + ip_range="$3" + + if ! ng_exists "${bridge_name}:"; then + ngctl -d -f - </dev/null 2>&1 +} + +function calculate_mac_address { + name="$1" + source=$(md5 -r -s "$name" | awk '{print $1}') + echo "${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}:${source:10:2}" +} + + +main "${@}" diff --git a/ansible/roles/bhyve/tasks/freebsd.yaml b/ansible/roles/bhyve/tasks/freebsd.yaml index dc5b243..7da5e07 100644 --- a/ansible/roles/bhyve/tasks/freebsd.yaml +++ b/ansible/roles/bhyve/tasks/freebsd.yaml @@ -13,6 +13,17 @@ - bhyve-firmware # For UEFI state: present +- name: Install scripts + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0755 + owner: root + group: wheel + loop: + - src: bhyve_netgraph_bridge.bash + dest: /usr/local/bin/bhyve_netgraph_bridge + - name: Create zfs dataset zfs: name: "{{ bhyve_dataset }}" diff --git a/ansible/roles/firewall/files/homeserver_pf.conf b/ansible/roles/firewall/files/homeserver_pf.conf index 9653cbf..75dd668 100644 --- a/ansible/roles/firewall/files/homeserver_pf.conf +++ b/ansible/roles/firewall/files/homeserver_pf.conf @@ -1,6 +1,7 @@ ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }" -jail_net_v4 = "10.193.223.0/24" -full_nat_v4 = "10.213.177.0/24" +not_ext_if = "{ !igb0 !igb1 !ix0 !ix1 !wlan0 }" +jail_nat_v4 = "{ 10.215.1.0/24 }" +not_jail_nat_v4 = "{ any, !10.215.1.0/24 }" dhcp = "{ bootpc, bootps }" allow = "{ wgh wgf }" @@ -15,23 +16,29 @@ unifi_ports = "{ 8443 3478 10001 8080 1900 8843 8880 6789 5514 }" set skip on lo # redirections -nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0) -nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0) +nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0) +rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53 -rdr pass on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53 -rdr pass on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53 +# cloak +nat pass on $ext_if inet from 10.215.2.0/24 to !10.215.2.0/24 -> (wlan0) +rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.2.1 port 53 -> 1.1.1.1 port 53 -nat pass on host_uplink0 inet proto tcp from any to any port 8081 tag ALLOWED -> (host_uplink0) -rdr pass on $ext_if inet proto tcp from any to any port 8081 tag ALLOWED -> 10.193.223.20 port 8081 +rdr pass on $ext_if inet proto tcp from any to any port 8081 -> 10.215.2.2 port 8081 # Forward ports for unifi controller -rdr pass on $ext_if inet proto tcp from any to any port 65022 tag ALLOWED -> 10.213.177.8 port 22 -rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports tag ALLOWED -> 10.213.177.8 +rdr pass on $ext_if inet proto tcp from any to any port 65022 -> 10.213.177.8 port 22 +rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports -> 10.213.177.8 # filtering block log all pass out on $ext_if +pass in on jail_nat +# Allow traffic from my machine to the jails/virtual machines +pass out on jail_nat from $jail_nat_v4 + +pass in on restricted_nat proto {udp, tcp} from any to any port { 53 51820 } + # We pass on the interfaces listed in allow rather than skipping on # them because changes to pass rules will update when running a # `service pf reload` but interfaces that we `skip` will not update (I @@ -44,9 +51,3 @@ pass on $ext_if proto icmp6 all pass in on $ext_if proto tcp to any port $tcp_pass_in pass in on $ext_if proto udp to any port $udp_pass_in - -pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp - -pass in on host_uplink0 proto udp from any to any port { 53 51820 } -pass out on host_uplink0 proto tcp from any to any port 8081 -pass on host_uplink1 diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf index 9dc57ec..559cf9a 100644 --- a/ansible/roles/firewall/files/odofreebsd_pf.conf +++ b/ansible/roles/firewall/files/odofreebsd_pf.conf @@ -1,6 +1,8 @@ -ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }" -jail_net_v4 = "10.193.223.0/24" -full_nat_v4 = "10.213.177.0/24" +ext_if = "{ wlan0 }" +not_ext_if = "{ !wlan0 }" +jail_nat_v4 = "{ 10.215.1.0/24 }" +not_jail_nat_v4 = "{ any, !10.215.1.0/24 }" +dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }" dhcp = "{ bootpc, bootps }" allow = "{ wgf wgh drmario colo }" @@ -14,22 +16,26 @@ udp_pass_in = "{ 53 51820 }" set skip on lo # redirections -nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0) -nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0) +nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0) +rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53 -rdr on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53 -rdr on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53 +# Redirect jaeger ports to virtual machine. +# nat pass on lo inet from 127.0.0.0/24 to 127.0.0.0/24 port {6831 6832 16686 14268} -> (jail_nat) +# rdr pass proto {tcp, udp} from jail_nat:network to 127.0.0.0/24 port {6831 6832 16686 14268} -> 10.215.1.201 # filtering block log all -pass quick on $ext_if tagged ALLOWED pass out on $ext_if +pass in on jail_nat +# Allow traffic from my machine to the jails/virtual machines +pass out on jail_nat from $jail_nat_v4 + # We pass on the interfaces listed in allow rather than skipping on # them because changes to pass rules will update when running a # `service pf reload` but interfaces that we `skip` will not update (I # forget if its from adding, removing, or both. TODO: test to figure -# it out) +# it out). Also skipped interfaces are not subject to nat/rdr rules. pass quick on $allow pass on $ext_if proto icmp all @@ -37,9 +43,3 @@ pass on $ext_if proto icmp6 all pass in on $ext_if proto tcp to any port $tcp_pass_in pass in on $ext_if proto udp to any port $udp_pass_in - -pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp - -pass in on host_uplink0 proto udp from any to any port { 53 51820 } -pass out on host_uplink0 proto tcp from any to any port 8081 -pass on host_uplink1 diff --git a/ansible/roles/jail/files/jail_netgraph_bridge.bash b/ansible/roles/jail/files/jail_netgraph_bridge.bash new file mode 100644 index 0000000..a17dbd0 --- /dev/null +++ b/ansible/roles/jail/files/jail_netgraph_bridge.bash @@ -0,0 +1,120 @@ +#!/usr/local/bin/bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function main { + if [ "$1" = "start" ]; then + shift 1 + start_jail "${@}" + elif [ "$1" = "stop" ]; then + shift 1 + stop_jail "${@}" + else + >&2 echo "Unrecognized command" + exit 1 + fi +} + +function start_jail { + host_interface_name="$1" + bridge_name="bridge_${host_interface_name}" + jail_interface_name="$2" + ip_range="$3" + + assert_bridge "$host_interface_name" "$bridge_name" "$ip_range" + + bridge_link_name=$(detect_available_link "${bridge_name}") + ngctl -d -f - </dev/null 2>&1 +} + +function ng_bridge_get_num_hooks { + ngctl show "${1}" | grep -oE 'Num hooks: [0-9]+' | sed 's/Num hooks: //g' +} + +function detect_available_link { + bridge_name="$1" + linknum=1 + while true; do + link_name="link${linknum}" + if ! ng_exists "${bridge_name}:${link_name}"; then + echo "$link_name" + return + fi + (>&2 echo "$link_name failed on $bridge_name") + linknum=$((linknum + 1)) + if [ "$linknum" -gt 90 ]; then + (>&2 echo "No available links on bridge $bridge_name") + exit 1 + fi + done +} + +function wait_for_interface_to_exist { + # Wait for a vnet interface to exist again as a jail is shutting + # down. If you delete the netgraph node before the interface + # device exists, then the interface device will persist in a + # broken state. + ifname="$1" + max_wait_seconds="$2" + start=$(date +%s) + while true; do + now=$(date +%s) + if [ $((now - start)) -gt $max_wait_seconds ]; then + (>&2 echo "Waited for at least $max_wait_seconds seconds but the interface $ifname did not appear.") + return 1; + fi + if ifconfig "$ifname" >/dev/null 2>&1; then + return 0; + fi + sleep 2 + done +} + +main "${@}" diff --git a/ansible/roles/jail/files/jails/cloak.conf b/ansible/roles/jail/files/jails/cloak.conf index 181974f..dac8985 100644 --- a/ansible/roles/jail/files/jails/cloak.conf +++ b/ansible/roles/jail/files/jails/cloak.conf @@ -1,8 +1,11 @@ cloak { - path = "/jail/main/jails/cloak"; + path = "/jail/main/jails/${name}"; vnet; - vnet.interface += "host_link2"; - vnet.interface += "wg_uplink0"; + exec.prestart += "/usr/local/bin/jail_netgraph_bridge start restricted_nat jail${name} 10.215.2.1/24"; + exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop restricted_nat jail${name}"; + vnet.interface += "jail${name}"; + vnet.interface += "cloak"; + devfs_ruleset = 13; mount.devfs; # To expose tun device diff --git a/ansible/roles/jail/files/jails/dagger.conf b/ansible/roles/jail/files/jails/dagger.conf index cab230e..97034c8 100644 --- a/ansible/roles/jail/files/jails/dagger.conf +++ b/ansible/roles/jail/files/jails/dagger.conf @@ -1,7 +1,7 @@ dagger { - path = "/jail/main/jails/dagger"; + path = "/jail/main/jails/${name}"; vnet; - vnet.interface += "wg_link2"; + vnet.interface += "dagger"; exec.start += "/bin/sh /etc/rc"; exec.stop = "/bin/sh /etc/rc.shutdown jail"; diff --git a/ansible/roles/jail/files/jails/nat_dhcp.conf b/ansible/roles/jail/files/jails/nat_dhcp.conf index c4f1ba6..17609e2 100644 --- a/ansible/roles/jail/files/jails/nat_dhcp.conf +++ b/ansible/roles/jail/files/jails/nat_dhcp.conf @@ -1,9 +1,12 @@ nat_dhcp { - path = "/jail/main/jails/nat_dhcp"; + path = "/jail/main/jails/${name}"; vnet; - vnet.interface += "host_link3"; + exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24"; + exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}"; + vnet.interface += "jail${name}"; + devfs_ruleset = 14; - mount.devfs; # To expose tun device + mount.devfs; exec.start += "/bin/sh /etc/rc"; exec.stop = "/bin/sh /etc/rc.shutdown jail"; diff --git a/ansible/roles/jail/files/jails/sample.conf b/ansible/roles/jail/files/jails/sample.conf new file mode 100644 index 0000000..2f0ee19 --- /dev/null +++ b/ansible/roles/jail/files/jails/sample.conf @@ -0,0 +1,14 @@ +sample { + path = "/jail/main/jails/${name}"; + vnet; + exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24"; + exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}"; + vnet.interface += "jail${name}"; + + devfs_ruleset = 14; + mount.devfs; + + exec.start += "/bin/sh /etc/rc"; + exec.stop = "/bin/sh /etc/rc.shutdown jail"; + exec.consolelog = "/var/log/jail_${name}_console.log"; +} diff --git a/ansible/roles/jail/files/setup_netgraph_homeserver b/ansible/roles/jail/files/setup_netgraph_homeserver index 1a0cef7..cfa53bf 100644 --- a/ansible/roles/jail/files/setup_netgraph_homeserver +++ b/ansible/roles/jail/files/setup_netgraph_homeserver @@ -1,77 +1,13 @@ #!/usr/local/bin/bash cleanup() { - ngctl shutdown host_link2: - ngctl shutdown host_uplink0: - ngctl shutdown host_bridge0: - ngctl shutdown wg_link2: - ngctl shutdown wg_uplink0: - ngctl shutdown wg_bridge0: - ngctl shutdown host_link3: - ngctl shutdown host_uplink1: - ngctl shutdown host_bridge1: + /usr/local/bin/jail_netgraph_bridge stop cloak dagger } setup_netgraph_start() { cleanup - # Create a bridge for jails that only speak wireguard - ngctl -d -f - <&2 echo "Unrecognized command" -fi diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml index 8d1f6bc..b34afe2 100644 --- a/ansible/roles/jail/tasks/freebsd.yaml +++ b/ansible/roles/jail/tasks/freebsd.yaml @@ -125,6 +125,17 @@ - src: "{{ netgraph_config }}" dest: /usr/local/bin/setup_netgraph +- name: Install scripts + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0755 + owner: root + group: wheel + loop: + - src: jail_netgraph_bridge.bash + dest: /usr/local/bin/jail_netgraph_bridge + - name: Enable setup_netgraph when: netgraph_config is defined community.general.sysrc: diff --git a/ansible/roles/jail_nat_dhcp/files/dhcpd.conf b/ansible/roles/jail_nat_dhcp/files/dhcpd.conf deleted file mode 100644 index aa36935..0000000 --- a/ansible/roles/jail_nat_dhcp/files/dhcpd.conf +++ /dev/null @@ -1,13 +0,0 @@ -# option definitions common to all supported networks... -option domain-name "home.arpa"; -# option domain-name-servers ns1.home.arpa; -option subnet-mask 255.255.255.0; -default-lease-time 600; -max-lease-time 7200; - -subnet 10.213.177.0 netmask 255.255.255.0 { - range 10.213.177.10 10.213.177.250; - option broadcast-address 10.213.177.255; - option routers 10.213.177.1; - option domain-name-servers 10.213.177.1; -} diff --git a/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf new file mode 100644 index 0000000..ce2043c --- /dev/null +++ b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf @@ -0,0 +1,31 @@ +{ + "Dhcp4": { + "interfaces-config": { + "interfaces": [ "jailnat_dhcp" ] + }, + "subnet4": [ + { + "subnet": "10.215.1.0/24", + "pools": [ { "pool": "10.215.1.10-10.215.1.200" } ], + "option-data": [ + { + "name": "routers", + "data": "10.215.1.1" + } + ], + "reservations": [ + { + "hw-address": "d6:19:4c:68:fc:c7", + "ip-address": "10.215.1.201" + } + ] + } + ], + "option-data": [ + { + "name": "domain-name-servers", + "data": "10.215.1.1" + } + ] + } +} diff --git a/ansible/roles/jail_nat_dhcp/files/rc.conf b/ansible/roles/jail_nat_dhcp/files/rc.conf index cfa78b1..e0d493a 100644 --- a/ansible/roles/jail_nat_dhcp/files/rc.conf +++ b/ansible/roles/jail_nat_dhcp/files/rc.conf @@ -1,3 +1,3 @@ -ifconfig_host_link3="inet 10.213.177.254 netmask 255.255.255.0" -defaultrouter="10.213.177.1" -dhcpd_enable="YES" +ifconfig_jailnat_dhcp="inet 10.215.1.254 netmask 255.255.255.0" +defaultrouter="10.215.1.1" +kea_enable="YES" diff --git a/ansible/roles/jail_nat_dhcp/files/resolv.conf b/ansible/roles/jail_nat_dhcp/files/resolv.conf index 582d17e..c21685a 100644 --- a/ansible/roles/jail_nat_dhcp/files/resolv.conf +++ b/ansible/roles/jail_nat_dhcp/files/resolv.conf @@ -1,2 +1,2 @@ search home.arpa -nameserver 10.213.177.1 +nameserver 10.215.1.1 diff --git a/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml b/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml index b98022f..0ce13f0 100644 --- a/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml +++ b/ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml @@ -1,7 +1,7 @@ - name: Install packages package: name: - - dhcpd + - kea state: present - name: Install Configuration @@ -14,7 +14,7 @@ loop: - src: rc.conf dest: /etc/rc.conf - - src: dhcpd.conf - dest: /usr/local/etc/dhcpd.conf + - src: kea-dhcp4.conf + dest: /usr/local/etc/kea/kea-dhcp4.conf - src: resolv.conf dest: /etc/resolv.conf diff --git a/ansible/roles/package_manager/tasks/freebsd.yaml b/ansible/roles/package_manager/tasks/freebsd.yaml index c98f0c0..ce9b8b3 100644 --- a/ansible/roles/package_manager/tasks/freebsd.yaml +++ b/ansible/roles/package_manager/tasks/freebsd.yaml @@ -8,6 +8,12 @@ loop: - /usr/local/etc/pkg - /usr/local/etc/pkg/repos + +- name: Install packages + package: + name: + - pkg-provides + state: present - name: Install Configuration register: changed_config @@ -26,9 +32,3 @@ # - name: Replace all packages with packages from new repo # command: pkg upgrade -f -y # when: changed_config.changed - -- name: Install packages - package: - name: - - pkg-provides - state: present