From f3980dc821f68bf5ba04bb28806c93d95388b7eb Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 13 Jun 2023 14:13:55 -0400
Subject: [PATCH] Add support for both NAT and RAW network interfaces in
 virtual machine.

This is used so metallb can announce public ip addresses.
---
 .../bhyve/files/bhyve_netgraph_bridge.bash    | 83 ++++++++++++-------
 1 file changed, 53 insertions(+), 30 deletions(-)

diff --git a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
index 6928836..6c3d2b2 100644
--- a/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
+++ b/ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash
@@ -21,17 +21,22 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 #   doas bhyve_netgraph_bridge start poudriere zdata/vm/poudriere /vm/poudriere /vm/iso/FreeBSD-13.2-RELEASE-amd64-bootonly.iso
 #   doas bhyve_netgraph_bridge start poudriere zdata/vm/poudriere /vm/poudriere
 
+: ${VERBOSE:="NO"} # or YES
 : ${CPU_CORES:="1"}
 : ${MEMORY:="1G"}
-: ${NETWORK:="NAT"} # or RAW
+: ${NETWORK:="NAT"} # or RAW or BOTH
 : ${IP_RANGE:="10.215.1.1/24"} # Ignored for RAW networks
 : ${INTERFACE_NAME:="jail_nat"} # or the external interface like lagg0 for RAW networks
 : ${BRIDGE_NAME:="bridge_$INTERFACE_NAME"} # or bridge_raw for RAW networks
 : ${VNC_ENABLE:="NO"}
 : ${VNC_LISTEN:="127.0.0.1:5900"}
 
+if [ "$VERBOSE" = "YES" ]; then
+    set -x
+fi
+
 function main {
-    cmd="$1"
+    local cmd="$1"
     shift 1
     if [ "$cmd" = "create-disk" ]; then
         create_disk "${@}"
@@ -50,9 +55,9 @@ function die {
 }
 
 function create_disk {
-    zfs_path="$1"
-    mount_path="$2"
-    gigabytes="$3"
+    local zfs_path="$1"
+    local mount_path="$2"
+    local gigabytes="$3"
     zfs create -o "mountpoint=$mount_path" "$zfs_path"
     cp /usr/local/share/edk2-bhyve/BHYVE_UEFI_VARS.fd "${mount_path}/"
     tee "${mount_path}/settings" <<EOF
@@ -67,30 +72,42 @@ EOF
 }
 
 function start_vm {
-    name="$1"
-    zfs_path="$2"
-    mount_path="$3"
-    mount_cd="${4:-}"
+    local name="$1"
+    local zfs_path="$2"
+    local mount_path="$3"
+    local mount_cd="${4:-}"
 
     if [ -e "${mount_path}/settings" ]; then
         source "${mount_path}/settings"
     fi
 
-    host_interface_name="$INTERFACE_NAME" # for raw, external interface
-    bridge_name="$BRIDGE_NAME"
-    ip_range="$IP_RANGE" # for raw this value does not matter
+    local host_interface_name="$INTERFACE_NAME" # for raw, external interface
+    local bridge_name="$BRIDGE_NAME"
+    local ip_range="$IP_RANGE" # for raw this value does not matter
 
-    mac_address=$(calculate_mac_address "$name")
+    local mac_address=$(calculate_mac_address "$name")
 
+    local additional_args=()
 
     if [ "$NETWORK" = "NAT" ]; then
         assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
     elif [ "$NETWORK" = "RAW" ]; then
         assert_raw "$host_interface_name" "$bridge_name"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+    elif [ "$NETWORK" = "BOTH" ]; then
+        assert_bridge "jail_nat" "$bridge_name" "$ip_range"
+        assert_raw "$host_interface_name" "bridge_raw"
+        local bridge_link_name=$(detect_available_link "${bridge_name}")
+        local raw_bridge_link_name=$(detect_available_link "bridge_raw")
+        local raw_mac_address=$(calculate_mac_address "${name}_raw")
+        additional_args+=("-s" "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}")
+        additional_args+=("-s" "3:0,virtio-net,netgraph,path=bridge_raw:,peerhook=${raw_bridge_link_name},mac=${raw_mac_address}")
     else
         die 1 "Unrecognized NETWORK type $NETWORK"
     fi
-    bridge_link_name=$(detect_available_link "${bridge_name}")
 
 
     # -H release the CPU when guest issues HLT instruction. Otherwise 100% of core will be consumed.
@@ -99,7 +116,6 @@ function start_vm {
             # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080 \
 
     # TODO: Look into using nmdm instead of stdio for serial console
-    additional_args=()
     if [ -n "$mount_cd" ]; then
         additional_args+=("-s" "3,ahci-cd,$mount_cd")
     fi
@@ -116,13 +132,12 @@ function start_vm {
             -H \
             -s 0,hostbridge \
             -s "4,nvme,/dev/zvol/${zfs_path}/disk0" \
-            -s "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}" \
             -s 30,xhci,tablet \
             -s 31,lpc -l com1,stdio \
             -l "bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,${mount_path}/BHYVE_UEFI_VARS.fd" \
             "${additional_args[@]}" \
             "$name"
-        exit_code=$?
+        local exit_code=$?
         set -e
         set +x
         if [ $exit_code -eq 0 ]; then
@@ -147,10 +162,10 @@ function start_vm {
 }
 
 function detect_available_link {
-    bridge_name="$1"
-    linknum=1
+    local bridge_name="$1"
+    local linknum=1
     while true; do
-        link_name="link${linknum}"
+        local link_name="link${linknum}"
         if ! ng_exists "${bridge_name}:${link_name}"; then
             echo "$link_name"
             return
@@ -164,9 +179,9 @@ function detect_available_link {
 }
 
 function assert_bridge {
-    host_interface_name="$1"
-    bridge_name="$2"
-    ip_range="$3"
+    local host_interface_name="$1"
+    local bridge_name="$2"
+    local ip_range="$3"
 
     if ! ng_exists "${bridge_name}:"; then
         ngctl -d -f - <<EOF
@@ -182,13 +197,13 @@ EOF
 }
 
 function assert_raw {
-    extif="$1"
-    bridge_name="$2"
+    local extif="$1"
+    local bridge_name="$2"
 
     kldload -n ng_bridge ng_eiface ng_ether
 
     if ! ng_exists "${bridge_name}:"; then
-        ngctl -d -f - <<EOF
+        ngctlcat <<EOF
 # Create a bridge.
 mkpeer $extif: bridge lower link0
 # Assign a name to the bridge.
@@ -210,19 +225,27 @@ function ng_exists {
 }
 
 function calculate_mac_address {
-    name="$1"
-    source=$(md5 -r -s "$name" | awk '{print $1}')
+    local name="$1"
+    local source=$(md5 -r -s "$name" | awk '{print $1}')
     echo "06:${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}"
 }
 
 function find_available_port {
-    start_port="$1"
-    port="$start_port"
+    local start_port="$1"
+    local port="$start_port"
     while true; do
         sockstat -P tcp -p 443
         port=$((port + 1))
     done
 }
 
+function ngctlcat {
+    if [ "$VERBOSE" = "YES" ]; then
+        tee /dev/tty | ngctl -d -f -
+    else
+        ngctl -d -f -
+    fi
+}
+
 
 main "${@}"