machine_setup/ansible/roles/firewall/files/mrmanager_pf.conf

ext_if = "lagg0"
not_ext_if = "{ !lagg0 }"
jail_nat_v4 = "{ 10.215.1.0/24 }"
not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
pub_k8s = "{ 74.80.180.136/29, !74.80.180.138 }"

dhcp = "{ bootpc, bootps }"
allow = "{ colo }"

tcp_pass_in = "{ 22 }"
udp_pass_in = "{ 53 51820 51821 51822 }"

# Rules must be in order: options, normalization, queueing, translation, filtering

# options
set skip on lo
set optimization conservative
set block-policy return
set fail-policy drop

# normalization
scrub in on $ext_if all fragment reassemble

# translation

# Evaluation order of the translation rules is dependent on the type
# of the translation rules and of the direction of a packet.  binat
# rules are always evaluated first.  Then either the rdr rules are
# evaluated on an inbound packet or the nat rules on an outbound
# packet.  Rules of the same type are evaluated in the same order in
# which they appear in the ruleset.  The first matching rule decides
# what action is taken.

# redirections
nat on $ext_if inet from ! ($ext_if) to ! ($ext_if) -> ($ext_if)
rdr pass on jail_nat proto {tcp, udp} from any to 10.215.1.1 port 53 tag REDIREXTERNAL -> 1.1.1.1 port 53

rdr pass on $ext_if proto {tcp, udp} to ($ext_if) port 6443 -> 10.215.1.204 port 6443
rdr pass on jail_nat proto {tcp, udp} to ($ext_if) port 6443 tag REDIRINTERNAL -> 10.215.1.204 port 6443

rdr pass on $ext_if proto {tcp, udp} to ($ext_if) port 19993 -> 10.215.1.204 port 19993
rdr pass on jail_nat proto {tcp, udp} to ($ext_if) port 19993 tag REDIRINTERNAL -> 10.215.1.204 port 19993

rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 65099 -> 10.215.1.210 port 22
rdr pass proto {tcp, udp} from $jail_nat_v4 to ($ext_if) port 65099 tag REDIRINTERNAL -> 10.215.1.210 port 22

rdr pass proto {tcp, udp} from $not_jail_nat_v4 to ($ext_if) port 53 -> 10.215.1.211 port 53
rdr pass proto {tcp, udp} from $jail_nat_v4 to ($ext_if) port 53 tag REDIRINTERNAL -> 10.215.1.211 port 53

nat pass tagged REDIRINTERNAL -> (jail_nat)
nat pass tagged REDIREXTERNAL -> ($ext_if)

# filtering
block log all
pass quick proto {icmp, icmp6} all
pass out quick on $ext_if
# We pass on the interfaces listed in allow rather than skipping on
# them because changes to pass rules will update when running a
# `service pf reload` but interfaces that we `skip` will not update (I
# forget if its from adding, removing, or both. TODO: test to figure
# it out). Also skipped interfaces are not subject to nat/rdr rules.
pass quick on $allow

# Single interface kubernetes cluster is working with the following run on mrmanager:
#   doas route add -host 74.80.180.139 -interface jail_nat
#   doas route add -net 10.129.0.0/16 -interface jail_nat
#   doas sysctl net.link.ether.inet.proxyall=1
# Plus this in pf.conf:
#   pass quick from any to 74.80.180.139
pass quick from any to $pub_k8s

pass in on jail_nat
# Allow traffic from my machine to the jails/virtual machines
pass out on jail_nat from (jail_nat:network)


pass in on $ext_if proto tcp to any port $tcp_pass_in
pass in on $ext_if proto udp to any port $udp_pass_in