Compare commits
3 Commits
67d8dc3f5c
...
8424abdfa9
Author | SHA1 | Date | |
---|---|---|---|
![]() |
8424abdfa9 | ||
![]() |
b9a199c5f5 | ||
![]() |
0c23b46426 |
81
ansible/roles/mrmanager/files/k8s_rollback.bash
Normal file
81
ansible/roles/mrmanager/files/k8s_rollback.bash
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Kill the virtual machines, take a snapshot, and start the virtual machines up again.
|
||||||
|
set -euo pipefail
|
||||||
|
IFS=$'\n\t'
|
||||||
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
|
|
||||||
|
: ${VMS:="poudriere controller0 controller1 controller2 worker0 worker1 worker2"}
|
||||||
|
|
||||||
|
############## Setup #########################
|
||||||
|
|
||||||
|
function die {
|
||||||
|
local status_code="$1"
|
||||||
|
shift
|
||||||
|
(>&2 echo "${@}")
|
||||||
|
exit "$status_code"
|
||||||
|
}
|
||||||
|
|
||||||
|
function log {
|
||||||
|
(>&2 echo "${@}")
|
||||||
|
}
|
||||||
|
|
||||||
|
############## Program #########################
|
||||||
|
|
||||||
|
function main {
|
||||||
|
convert_to_array 'VMS'
|
||||||
|
local snapshot_name="$1"
|
||||||
|
|
||||||
|
local rollback_targets=$(doas zfs list -t snapshot -p -o name | grep -E "@${snapshot_name}\$")
|
||||||
|
log "Rolling back to the following snapshots:"
|
||||||
|
while read target; do
|
||||||
|
log " $target"
|
||||||
|
done<<<"$rollback_targets"
|
||||||
|
sanity_check_targets "$rollback_targets"
|
||||||
|
|
||||||
|
kill_all_bhyve
|
||||||
|
log "Rolling back to snapshot $1"
|
||||||
|
while read target; do
|
||||||
|
doas zfs rollback "$target"
|
||||||
|
done<<<"$rollback_targets"
|
||||||
|
launch_all_bhyve
|
||||||
|
log "Done."
|
||||||
|
}
|
||||||
|
|
||||||
|
function kill_all_bhyve {
|
||||||
|
log "Killing all virtual machines."
|
||||||
|
doas killall bhyve; while true; do tmux ls || break; sleep 2; done;
|
||||||
|
}
|
||||||
|
|
||||||
|
function launch_all_bhyve {
|
||||||
|
log "Launching all virtual machines."
|
||||||
|
for vm in "${VMS[@]}"; do log "starting $vm"; tmux new -d -s $vm doas bhyve_netgraph_bridge start $vm zdata/vm/$vm /vm/$vm; sleep 5; done
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanity_check_targets {
|
||||||
|
local rollback_targets="$1"
|
||||||
|
local targets_length=$(wc -l <<<"$rollback_targets")
|
||||||
|
if [ $targets_length -ne 7 ]; then
|
||||||
|
die 1 "Expecting the snapshot to exist for each k8s vm and the k8s persistent storage space. Only found $targets_length targets."
|
||||||
|
fi
|
||||||
|
while read rollback_target; do
|
||||||
|
if [[ ! $rollback_target = "zdata/vm"[/@]* ]] && [[ ! $rollback_target = "zdata/k8spersistent"[/@]* ]]; then
|
||||||
|
die 1 "Rollback target not in k8s-vm-related zfs datasets: $rollback_target"
|
||||||
|
fi
|
||||||
|
done<<<"$rollback_targets"
|
||||||
|
}
|
||||||
|
|
||||||
|
function convert_to_array {
|
||||||
|
# Converts a string separated by a value into an array. An example use-case would be for parsing environment variables like $PATH.
|
||||||
|
#
|
||||||
|
# WARNING: This uses nameref to modify the env variable at the first parameter, making this function very much not "functional". It does not return anything on stdout so it cannot be used with $().
|
||||||
|
local -n string_to_convert="$1"
|
||||||
|
local ifs_in_string="${2:- }"
|
||||||
|
local current_ifs="$IFS"
|
||||||
|
IFS="$ifs_in_string"
|
||||||
|
string_to_convert=($string_to_convert)
|
||||||
|
# Restore IFS to the original value in case this function is not run inside a subshell
|
||||||
|
IFS="$current_ifs"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
@ -5,6 +5,9 @@ set -euo pipefail
|
|||||||
IFS=$'\n\t'
|
IFS=$'\n\t'
|
||||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
|
|
||||||
|
: ${VMS:="poudriere controller0 controller1 controller2 worker0 worker1 worker2"}
|
||||||
|
: ${WORKERS:="controller0 controller1 controller2 worker0 worker1 worker2"}
|
||||||
|
|
||||||
############## Setup #########################
|
############## Setup #########################
|
||||||
|
|
||||||
function die {
|
function die {
|
||||||
@ -21,10 +24,12 @@ function log {
|
|||||||
############## Program #########################
|
############## Program #########################
|
||||||
|
|
||||||
function main {
|
function main {
|
||||||
snapshot_name="$1"
|
convert_to_array 'VMS'
|
||||||
|
convert_to_array 'WORKERS'
|
||||||
|
local snapshot_name="$1"
|
||||||
kill_all_bhyve
|
kill_all_bhyve
|
||||||
log "Taking snapshot $1"
|
log "Taking snapshot $1"
|
||||||
for worker in worker0 worker1 worker2 controller0 controller1 controller2; do
|
for worker in "${WORKERS[@]}"; do
|
||||||
doas zfs snapshot -r zdata/vm/${worker}/disk0@${snapshot_name}
|
doas zfs snapshot -r zdata/vm/${worker}/disk0@${snapshot_name}
|
||||||
done
|
done
|
||||||
doas zfs snapshot zdata/k8spersistent@${snapshot_name}
|
doas zfs snapshot zdata/k8spersistent@${snapshot_name}
|
||||||
@ -39,7 +44,20 @@ function kill_all_bhyve {
|
|||||||
|
|
||||||
function launch_all_bhyve {
|
function launch_all_bhyve {
|
||||||
log "Launching all virtual machines."
|
log "Launching all virtual machines."
|
||||||
for vm in poudriere controller0 controller1 controller2 worker0 worker1 worker2; do tmux new -d -s $vm doas bhyve_netgraph_bridge start $vm zdata/vm/$vm /vm/$vm; sleep 5; done
|
for vm in "${VMS[@]}"; do log "starting $vm"; tmux new -d -s $vm doas bhyve_netgraph_bridge start $vm zdata/vm/$vm /vm/$vm; sleep 5; done
|
||||||
|
}
|
||||||
|
|
||||||
|
function convert_to_array {
|
||||||
|
# Converts a string separated by a value into an array. An example use-case would be for parsing environment variables like $PATH.
|
||||||
|
#
|
||||||
|
# WARNING: This uses nameref to modify the env variable at the first parameter, making this function very much not "functional". It does not return anything on stdout so it cannot be used with $().
|
||||||
|
local -n string_to_convert="$1"
|
||||||
|
local ifs_in_string="${2:- }"
|
||||||
|
local current_ifs="$IFS"
|
||||||
|
IFS="$ifs_in_string"
|
||||||
|
string_to_convert=($string_to_convert)
|
||||||
|
# Restore IFS to the original value in case this function is not run inside a subshell
|
||||||
|
IFS="$current_ifs"
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
@ -48,5 +48,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: wheel
|
group: wheel
|
||||||
loop:
|
loop:
|
||||||
- src: "k8s_snapshot.bash"
|
- src: k8s_snapshot.bash
|
||||||
dest: /usr/local/bin/k8s_snapshot
|
dest: /usr/local/bin/k8s_snapshot
|
||||||
|
- src: k8s_rollback.bash
|
||||||
|
dest: /usr/local/bin/k8s_rollback
|
||||||
|
Loading…
x
Reference in New Issue
Block a user