diff --git a/ansible/roles/mrmanager/files/k8s_rollback.bash b/ansible/roles/mrmanager/files/k8s_rollback.bash new file mode 100644 index 0000000..bad2f69 --- /dev/null +++ b/ansible/roles/mrmanager/files/k8s_rollback.bash @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Kill the virtual machines, take a snapshot, and start the virtual machines up again. +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: ${VMS:="poudriere controller0 controller1 controller2 worker0 worker1 worker2"} + +############## Setup ######################### + +function die { + local status_code="$1" + shift + (>&2 echo "${@}") + exit "$status_code" +} + +function log { + (>&2 echo "${@}") +} + +############## Program ######################### + +function main { + VMS=($(convert_ifs "$VMS" ' ' "$IFS")) + local snapshot_name="$1" + + local rollback_targets=$(doas zfs list -t snapshot -p -o name | grep -E "@${snapshot_name}\$") + log "Rolling back to the following snapshots:" + while read target; do + log " $target" + done<<<"$rollback_targets" + sanity_check_targets "$rollback_targets" + + kill_all_bhyve + log "Rolling back to snapshot $1" + while read target; do + zfs rollback "$target" + done<<<"$rollback_targets" + launch_all_bhyve + log "Done." +} + +function kill_all_bhyve { + log "Killing all virtual machines." + doas killall bhyve; while true; do tmux ls || break; sleep 2; done; +} + +function launch_all_bhyve { + log "Launching all virtual machines." + for vm in $VMS[@]; do tmux new -d -s $vm doas bhyve_netgraph_bridge start $vm zdata/vm/$vm /vm/$vm; sleep 5; done +} + +function sanity_check_targets { + local rollback_targets="$1" + local targets_length=$(wc -l <<<"$rollback_targets") + if [ $targets_length -ne 7 ]; then + die 1 "Expecting the snapshot to exist for each k8s vm and the k8s persistent storage space. Only found $targets_length targets." + fi + while read rollback_target; do + if [[ ! $rollback_target = "zdata/vm"[/@]* ]] && [[ ! $rollback_target = "zdata/k8spersistent"[/@]* ]]; then + die 1 "Rollback target not in k8s-vm-related zfs datasets: $rollback_target" + fi + done<<<"$rollback_targets" +} + +function convert_ifs { + # Converts a string from one internal field separator to another. For example "foo bar baz" to "foo:bar:baz". This is useful for parsing a string as an array (for example, to read the value from environment variables like $PATH) while using a different IFS + # + # TODO: It would be a lot better if we could just convert from string to array directly using a temporarily different IFS variable without converting to a different string format first. + local string_to_convert="$1" + local ifs_in_string="$2" + local target_ifs="$3" + local current_ifs="$IFS" + IFS="$ifs_in_string" + local split_to_array=($string_to_convert) + local expanded_string=$(IFS="$target_ifs"; printf '%s' "${split_to_array[*]}") + # Restore IFS to the original value in case this function is not run inside a subshell + IFS="$current_ifs" + cat <<<"$expanded_string" +} + +main "$@" diff --git a/ansible/roles/mrmanager/tasks/freebsd.yaml b/ansible/roles/mrmanager/tasks/freebsd.yaml index 1834352..cff92d5 100644 --- a/ansible/roles/mrmanager/tasks/freebsd.yaml +++ b/ansible/roles/mrmanager/tasks/freebsd.yaml @@ -48,5 +48,7 @@ owner: root group: wheel loop: - - src: "k8s_snapshot.bash" + - src: k8s_snapshot.bash dest: /usr/local/bin/k8s_snapshot + - src: k8s_rollback.bash + dest: /usr/local/bin/k8s_rollback