From ba7567ad9c0a7e96fc628fc8e4f35df1ee980f5c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 25 Apr 2023 18:33:19 -0400 Subject: [PATCH] Switch to using a script to dynamically spin up the netgraph bridge for jails. --- .../environments/laptop/host_vars/odofreebsd | 1 - .../roles/firewall/files/odofreebsd_pf.conf | 23 ++-- .../jail/files/jail_netgraph_bridge.bash | 120 ++++++++++++++++++ ansible/roles/jail/files/jails/nat_dhcp.conf | 9 +- ansible/roles/jail/files/jails/sample.conf | 14 ++ ansible/roles/jail/files/setup_netgraph_odo | 87 ------------- ansible/roles/jail/tasks/freebsd.yaml | 11 ++ 7 files changed, 159 insertions(+), 106 deletions(-) create mode 100644 ansible/roles/jail/files/jail_netgraph_bridge.bash create mode 100644 ansible/roles/jail/files/jails/sample.conf delete mode 100644 ansible/roles/jail/files/setup_netgraph_odo diff --git a/ansible/environments/laptop/host_vars/odofreebsd b/ansible/environments/laptop/host_vars/odofreebsd index 6a44b37..f402874 100644 --- a/ansible/environments/laptop/host_vars/odofreebsd +++ b/ansible/environments/laptop/host_vars/odofreebsd @@ -10,7 +10,6 @@ pflog_conf: network_rc: "odofreebsd_network.conf" rc_conf: "odofreebsd_rc.conf" loader_conf: "odofreebsd_loader.conf" -netgraph_config: "setup_netgraph_odo" install_graphics: true graphics_driver: "intel" cputype: "intel" diff --git a/ansible/roles/firewall/files/odofreebsd_pf.conf b/ansible/roles/firewall/files/odofreebsd_pf.conf index 9dc57ec..89f1832 100644 --- a/ansible/roles/firewall/files/odofreebsd_pf.conf +++ b/ansible/roles/firewall/files/odofreebsd_pf.conf @@ -1,6 +1,8 @@ -ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }" -jail_net_v4 = "10.193.223.0/24" -full_nat_v4 = "10.213.177.0/24" +ext_if = "{ wlan0 }" +not_ext_if = "{ !wlan0 }" +jail_nat_v4 = "{ 10.193.223.0/24 10.213.177.0/24 10.215.1.0/24 }" +not_jail_nat_v4 = "{ any, !10.193.223.0/24 !10.213.177.0/24 !10.215.1.0/24 }" +dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }" dhcp = "{ bootpc, bootps }" allow = "{ wgf wgh drmario colo }" @@ -14,16 +16,13 @@ udp_pass_in = "{ 53 51820 }" set skip on lo # redirections -nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0) -nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0) - -rdr on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53 -rdr on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53 +nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0) +rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53 # filtering block log all -pass quick on $ext_if tagged ALLOWED pass out on $ext_if +pass in on jail_nat # We pass on the interfaces listed in allow rather than skipping on # them because changes to pass rules will update when running a @@ -37,9 +36,3 @@ pass on $ext_if proto icmp6 all pass in on $ext_if proto tcp to any port $tcp_pass_in pass in on $ext_if proto udp to any port $udp_pass_in - -pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp - -pass in on host_uplink0 proto udp from any to any port { 53 51820 } -pass out on host_uplink0 proto tcp from any to any port 8081 -pass on host_uplink1 diff --git a/ansible/roles/jail/files/jail_netgraph_bridge.bash b/ansible/roles/jail/files/jail_netgraph_bridge.bash new file mode 100644 index 0000000..a17dbd0 --- /dev/null +++ b/ansible/roles/jail/files/jail_netgraph_bridge.bash @@ -0,0 +1,120 @@ +#!/usr/local/bin/bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function main { + if [ "$1" = "start" ]; then + shift 1 + start_jail "${@}" + elif [ "$1" = "stop" ]; then + shift 1 + stop_jail "${@}" + else + >&2 echo "Unrecognized command" + exit 1 + fi +} + +function start_jail { + host_interface_name="$1" + bridge_name="bridge_${host_interface_name}" + jail_interface_name="$2" + ip_range="$3" + + assert_bridge "$host_interface_name" "$bridge_name" "$ip_range" + + bridge_link_name=$(detect_available_link "${bridge_name}") + ngctl -d -f - </dev/null 2>&1 +} + +function ng_bridge_get_num_hooks { + ngctl show "${1}" | grep -oE 'Num hooks: [0-9]+' | sed 's/Num hooks: //g' +} + +function detect_available_link { + bridge_name="$1" + linknum=1 + while true; do + link_name="link${linknum}" + if ! ng_exists "${bridge_name}:${link_name}"; then + echo "$link_name" + return + fi + (>&2 echo "$link_name failed on $bridge_name") + linknum=$((linknum + 1)) + if [ "$linknum" -gt 90 ]; then + (>&2 echo "No available links on bridge $bridge_name") + exit 1 + fi + done +} + +function wait_for_interface_to_exist { + # Wait for a vnet interface to exist again as a jail is shutting + # down. If you delete the netgraph node before the interface + # device exists, then the interface device will persist in a + # broken state. + ifname="$1" + max_wait_seconds="$2" + start=$(date +%s) + while true; do + now=$(date +%s) + if [ $((now - start)) -gt $max_wait_seconds ]; then + (>&2 echo "Waited for at least $max_wait_seconds seconds but the interface $ifname did not appear.") + return 1; + fi + if ifconfig "$ifname" >/dev/null 2>&1; then + return 0; + fi + sleep 2 + done +} + +main "${@}" diff --git a/ansible/roles/jail/files/jails/nat_dhcp.conf b/ansible/roles/jail/files/jails/nat_dhcp.conf index c4f1ba6..17609e2 100644 --- a/ansible/roles/jail/files/jails/nat_dhcp.conf +++ b/ansible/roles/jail/files/jails/nat_dhcp.conf @@ -1,9 +1,12 @@ nat_dhcp { - path = "/jail/main/jails/nat_dhcp"; + path = "/jail/main/jails/${name}"; vnet; - vnet.interface += "host_link3"; + exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24"; + exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}"; + vnet.interface += "jail${name}"; + devfs_ruleset = 14; - mount.devfs; # To expose tun device + mount.devfs; exec.start += "/bin/sh /etc/rc"; exec.stop = "/bin/sh /etc/rc.shutdown jail"; diff --git a/ansible/roles/jail/files/jails/sample.conf b/ansible/roles/jail/files/jails/sample.conf new file mode 100644 index 0000000..2f0ee19 --- /dev/null +++ b/ansible/roles/jail/files/jails/sample.conf @@ -0,0 +1,14 @@ +sample { + path = "/jail/main/jails/${name}"; + vnet; + exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24"; + exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}"; + vnet.interface += "jail${name}"; + + devfs_ruleset = 14; + mount.devfs; + + exec.start += "/bin/sh /etc/rc"; + exec.stop = "/bin/sh /etc/rc.shutdown jail"; + exec.consolelog = "/var/log/jail_${name}_console.log"; +} diff --git a/ansible/roles/jail/files/setup_netgraph_odo b/ansible/roles/jail/files/setup_netgraph_odo deleted file mode 100644 index 1a0cef7..0000000 --- a/ansible/roles/jail/files/setup_netgraph_odo +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/local/bin/bash - -cleanup() { - ngctl shutdown host_link2: - ngctl shutdown host_uplink0: - ngctl shutdown host_bridge0: - ngctl shutdown wg_link2: - ngctl shutdown wg_uplink0: - ngctl shutdown wg_bridge0: - ngctl shutdown host_link3: - ngctl shutdown host_uplink1: - ngctl shutdown host_bridge1: -} - -setup_netgraph_start() { - cleanup - - # Create a bridge for jails that only speak wireguard - ngctl -d -f - <&2 echo "Unrecognized command" -fi diff --git a/ansible/roles/jail/tasks/freebsd.yaml b/ansible/roles/jail/tasks/freebsd.yaml index 8d1f6bc..b34afe2 100644 --- a/ansible/roles/jail/tasks/freebsd.yaml +++ b/ansible/roles/jail/tasks/freebsd.yaml @@ -125,6 +125,17 @@ - src: "{{ netgraph_config }}" dest: /usr/local/bin/setup_netgraph +- name: Install scripts + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0755 + owner: root + group: wheel + loop: + - src: jail_netgraph_bridge.bash + dest: /usr/local/bin/jail_netgraph_bridge + - name: Enable setup_netgraph when: netgraph_config is defined community.general.sysrc: