Snapshots and rht-vmctl

Phase 12 · Snapshots and rht-vmctl (~10 min)

• Install rht-vmctl.
• Verify the build and take the clean baseline.

Host: install the script

Open an empty file for the script. Run sudo -v first so the password prompt is out of the way; otherwise the paste would land in the password prompt.

sudo -v                                       # enter your password now
sudo tee /usr/local/bin/rht-vmctl >/dev/null

It now waits for input. Copy the whole script below with its Copy button, paste it into the terminal, press Enter, then Ctrl+D to save. The same command overwrites an older copy. Prefer an editor? sudo nano /usr/local/bin/rht-vmctl, paste, save with Ctrl+O and exit with Ctrl+X.

Download the script. Read it before running it. The expandable example below matches the download.

#!/usr/bin/env bash
# rht-vmctl - classroom-style VM control for the Kernel Path LXD home lab
# Install: sudo install -m 755 rht-vmctl /usr/local/bin/rht-vmctl
# Version 2 (2026-09-30): the servers' extra disks are snapshotted and reset too
# Version 3 (2026-10-04): 'servers' and 'all' mean the lab's VMs that exist, so the same
#   commands work in the system administration lab (servera, serverb) and the Ansible lab
set -uo pipefail

VERSION=3

PROJECT="${LAB_PROJECT:-rhce}"
BASELINE="${LAB_BASELINE:-clean}"
SERVERS=(servera serverb serverc serverd)

L()    { lxc --project "$PROJECT" "$@"; }
say()  { printf '\033[1;36m==>\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m!!\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1;31mxx\033[0m %s\n' "$*" >&2; exit 1; }

exists()   { L info "$1" >/dev/null 2>&1; }
# 'lxc query' ignores --project, so the project goes in the URL
Q()        { lxc query "$1$([[ $1 == *\?* ]] && echo '&' || echo '?')project=$PROJECT"; }
has_snap() { Q "/1.0/instances/$1/snapshots/$2" >/dev/null 2>&1; }
state()    { L list "^${1}\
Loading…
  quot; -c s --format csv; }

# Extra disks (servera-disk2 ...) are separate volumes: instance snapshots don't
# include them, so each gets its own snapshot with the same name.
vols() {
  Q "/1.0/instances/$1" 2>/dev/null | python3 -c '
import sys, json
for d in json.load(sys.stdin).get("expanded_devices", {}).values():
    if d.get("type") == "disk" and d.get("pool") and d.get("source") and d.get("path") != "/":
        print(d["pool"], d["source"])'
}
vol_has_snap() { Q "/1.0/storage-pools/$1/volumes/custom/$2/snapshots/$3" >/dev/null 2>&1; }

# Put a VM's extra disks back to snapshot NAME (the VM must be stopped)
restore_vols() {
  local v="$1" name="$2" pool vol
  while read -r pool vol; do
    [[ -n $vol ]] || continue
    if ! vol_has_snap "$pool" "$vol" "$name"; then
      warn "$vol has no '$name' snapshot, so it was not reset (run: rht-vmctl save $v)"
    elif L storage volume restore "$pool" "$vol" "$name" </dev/null; then
      echo "  $vol reset"
    else
      fail "$vol: restore failed (see 'The ZFS snapshot rule' in the guide)"
    fi
  done < <(vols "$v")
}

# Snapshots taken after NAME. ZFS only restores a VM's newest snapshot.
newer_than() {
  Q "/1.0/instances/$1/snapshots?recursion=1" 2>/dev/null | python3 -c '
import sys, json
names = [s["name"] for s in sorted(json.load(sys.stdin), key=lambda s: s["created_at"])]
if sys.argv[1] in names: print(" ".join(names[names.index(sys.argv[1]) + 1:]))' "$2"
}
restore_failed() {
  local newer; newer=$(newer_than "$1" "$2")
  if [[ -n $newer ]]; then
    fail "$1: newer snapshot(s) exist: $newer. ZFS only restores the newest; remove them first: rht-vmctl rmsnap $1 NAME"
  else
    fail "$1: restore failed"
  fi
}

# Expand targets: a VM name, 'servers' (those of servera-serverd that exist) or 'all'
# (workstation, the servers and utility, when they exist)
targets() {
  local t s out=()
  for t in "$@"; do
    case "$t" in
      servers) for s in "${SERVERS[@]}"; do exists "$s" && out+=("$s"); done ;;
      all)     for s in workstation "${SERVERS[@]}" utility; do exists "$s" && out+=("$s"); done ;;
      *)       out+=("$t") ;;
    esac
  done
  for t in "${out[@]}"; do
    if exists "$t"; then echo "$t"; else warn "no such VM: $t (skipped)"; fi
  done
}

RC=0
fail() { warn "$1"; RC=1; }

# Wait until a VM has booted: the LXD agent answers, then systemd finishes starting up.
# Classroom rht-vmctl hands back a machine that is ready to use; this does the same.
wait_ready() {
  local v="$1" t=0 limit="${LAB_WAIT:-180}"
  [[ ${LAB_NOWAIT:-0} == 1 ]] && return 0
  until L exec "$v" -- true >/dev/null 2>&1; do
    sleep 2; t=$((t + 2))
    (( t >= limit )) && { fail "$v: not ready after ${limit}s (check: rht-vmctl view $v)"; return 1; }
  done
  L exec "$v" -- systemctl is-system-running --wait >/dev/null 2>&1
  echo "  $v ready"
}

need_target() { [[ $# -gt 0 ]] || die "which VM? e.g. 'rht-vmctl $SUB servera' or 'rht-vmctl $SUB all'"; }
confirm()     { local a; read -r -p "$1 [y/N] " a; [[ $a == [yY]* ]]; }
has_ws()      { printf '%s\n' "$@" | grep -qx workstation; }

usage() {
cat <<USAGE
rht-vmctl v$VERSION - Kernel Path home lab (LXD project: $PROJECT)

Classroom commands (same syntax):
  rht-vmctl status    VM|all       show state
  rht-vmctl start     VM|all       start
  rht-vmctl stop      VM|all       clean shutdown
  rht-vmctl poweroff  VM|all       force off
  rht-vmctl reset     VM|all       back to the '$BASELINE' snapshot, extra disks included
  rht-vmctl fullreset VM|all       same as reset in this lab
  rht-vmctl view      VM           text console (Ctrl+a then q to leave)

Home-lab extras:
  rht-vmctl save    [VM|all] [-n NAME]   take a snapshot (default: all, '$BASELINE')
  rht-vmctl snaps   [VM|all]             list snapshots, extra disks included
  rht-vmctl restore VM NAME              restore a named snapshot
  rht-vmctl rmsnap  VM NAME              delete a snapshot
  rht-vmctl login   VM [USER]            shell in a VM (default user: student)
  rht-vmctl ws                           shortcut for student@workstation

VM can also be 'servers' (servera to serverd, whichever exist) or 'all'.
start/reset/restore wait until each VM has booted, like the classroom.
  LAB_NOWAIT=1 rht-vmctl reset servera   # return immediately instead
Anything that resets workstation asks first: your playbooks live there.
USAGE
}

cmd_status() {
  need_target "$@"
  local v; mapfile -t vms < <(targets "$@")
  for v in "${vms[@]}"; do printf '%-12s %s\n' "$v" "$(state "$v")"; done
}

cmd_start() {
  need_target "$@"
  local v; mapfile -t vms < <(targets "$@")
  for v in "${vms[@]}"; do
    [[ $(state "$v") == RUNNING ]] && { echo "  $v already running"; continue; }
    if L start "$v"; then echo "  $v started"; else fail "$v: start failed"; fi
  done
  for v in "${vms[@]}"; do wait_ready "$v"; done
}

cmd_stop() {
  need_target "$@"
  local v; mapfile -t vms < <(targets "$@")
  for v in "${vms[@]}"; do
    [[ $(state "$v") == STOPPED ]] && { echo "  $v already stopped"; continue; }
    L stop "$v" && echo "  $v stopped"
  done
}

cmd_poweroff() {
  need_target "$@"
  local v; mapfile -t vms < <(targets "$@")
  for v in "${vms[@]}"; do
    [[ $(state "$v") == STOPPED ]] && { echo "  $v already off"; continue; }
    L stop --force "$v" && echo "  $v powered off"
  done
}

cmd_reset() {
  need_target "$@"
  local v done_vms=(); mapfile -t vms < <(targets "$@")
  [[ ${#vms[@]} -eq 0 ]] && die "nothing to reset"
  if has_ws "${vms[@]}"; then
    warn "this resets workstation too; your playbooks in ~/ansible will be lost"
    confirm "Continue?" || die "cancelled"
  fi
  say "resetting to '$BASELINE': ${vms[*]}"
  for v in "${vms[@]}"; do
    has_snap "$v" "$BASELINE" || { fail "$v has no '$BASELINE' snapshot (run: rht-vmctl save $v)"; continue; }
    [[ $(state "$v") == STOPPED ]] || L stop --force "$v"
    if L restore "$v" "$BASELINE"; then
      echo "  $v reset"
      restore_vols "$v" "$BASELINE"
      L start "$v"
      done_vms+=("$v")
    else
      restore_failed "$v" "$BASELINE"
      L start "$v" >/dev/null 2>&1 && done_vms+=("$v")
    fi
  done
  for v in "${done_vms[@]}"; do wait_ready "$v"; done
}

cmd_view() {
  [[ $# -eq 1 ]] || die "usage: rht-vmctl view VM"
  exists "$1" || die "no such VM: $1"
  L console "$1"
}

cmd_save() {
  local name="$BASELINE" args=() v over=()
  while [[ $# -gt 0 ]]; do
    case "$1" in
      -n|--name) name="${2:?missing snapshot name}"; shift 2 ;;
      *) args+=("$1"); shift ;;
    esac
  done
  [[ ${#args[@]} -eq 0 ]] && args=(all)
  mapfile -t vms < <(targets "${args[@]}")
  [[ ${#vms[@]} -eq 0 ]] && die "nothing to snapshot"
  for v in "${vms[@]}"; do has_snap "$v" "$name" && over+=("$v"); done
  if [[ ${#over[@]} -gt 0 ]]; then
    warn "snapshot '$name' already exists on: ${over[*]}"
    confirm "Replace it?" || die "cancelled"
  fi
  say "snapshot '$name' on: ${vms[*]}"
  for v in "${vms[@]}"; do
    has_snap "$v" "$name" && L delete "$v/$name"
    [[ $(state "$v") == RUNNING ]] && L exec "$v" -- sync </dev/null >/dev/null 2>&1   # flush the guest's disk cache
    if L snapshot "$v" "$name" </dev/null; then echo "  $v saved"; else fail "$v: snapshot failed"; continue; fi
    while read -r pool vol; do
      [[ -n $vol ]] || continue
      vol_has_snap "$pool" "$vol" "$name" && L storage volume delete "$pool" "$vol/$name" </dev/null >/dev/null
      if L storage volume snapshot "$pool" "$vol" "$name" </dev/null >/dev/null; then echo "  $vol saved"
      else fail "$vol: snapshot failed"; fi
    done < <(vols "$v")
  done
}

cmd_snaps() {
  local v; [[ $# -eq 0 ]] && set -- all
  mapfile -t vms < <(targets "$@")
  for v in "${vms[@]}"; do
    printf '\033[1m%s\033[0m\n' "$v"
    Q "/1.0/instances/$v/snapshots?recursion=1" 2>/dev/null | python3 -c '
import sys, json
raw = sys.stdin.read().strip()
d = json.loads(raw) if raw else []
if not d: print("  (none)")
for s in d: print("  %-20s %s" % (s["name"], s["created_at"][:19].replace("T", " ")))'
    while read -r pool vol; do
      [[ -n $vol ]] || continue
      printf '  disk %-15s ' "$vol"
      Q "/1.0/storage-pools/$pool/volumes/custom/$vol/snapshots" </dev/null 2>/dev/null | python3 -c '
import sys, json
raw = sys.stdin.read().strip()
names = [u.split("/snapshots/")[1].split("?")[0] for u in (json.loads(raw) if raw else [])]
print(", ".join(names) or "(none)")'
    done < <(vols "$v")
  done
}

cmd_restore() {
  [[ $# -eq 2 ]] || die "usage: rht-vmctl restore VM NAME"
  exists "$1"      || die "no such VM: $1"
  has_snap "$1" "$2" || die "$1 has no snapshot '$2' (see: rht-vmctl snaps $1)"
  if [[ $1 == workstation ]]; then confirm "Restore workstation to '$2'? Newer work is lost." || die "cancelled"; fi
  [[ $(state "$1") == STOPPED ]] || L stop --force "$1"
  if L restore "$1" "$2"; then
    say "$1 restored to '$2'"
    restore_vols "$1" "$2"
    L start "$1"
    wait_ready "$1"
  else
    restore_failed "$1" "$2"
    L start "$1" >/dev/null 2>&1
  fi
}

cmd_rmsnap() {
  [[ $# -eq 2 ]] || die "usage: rht-vmctl rmsnap VM NAME"
  has_snap "$1" "$2" || die "$1 has no snapshot '$2'"
  L delete "$1/$2" && say "deleted $1/$2"
  local pool vol
  while read -r pool vol; do
    [[ -n $vol ]] && vol_has_snap "$pool" "$vol" "$2" && L storage volume delete "$pool" "$vol/$2" </dev/null >/dev/null && say "deleted $vol/$2"
  done < <(vols "$1")
}

cmd_login() {
  [[ $# -ge 1 ]] || die "usage: rht-vmctl login VM [USER]"
  local vm="$1" user="${2:-student}"
  exists "$vm" || die "no such VM: $vm"
  if [[ $user == root ]]; then L exec "$vm" -- bash -l
  else L exec "$vm" -- su - "$user"; fi
}

SUB="${1:-help}"; shift || true
case "$SUB" in
  status)            cmd_status "$@" ;;
  start)             cmd_start "$@" ;;
  stop)              cmd_stop "$@" ;;
  poweroff)          cmd_poweroff "$@" ;;
  reset|fullreset)   cmd_reset "$@" ;;
  view|console)      cmd_view "$@" ;;
  save|snap)         cmd_save "$@" ;;
  snaps)             cmd_snaps "$@" ;;
  restore)           cmd_restore "$@" ;;
  rmsnap)            cmd_rmsnap "$@" ;;
  login)             cmd_login "$@" ;;
  ws)                cmd_login workstation student ;;
  help|-h|--help)    usage ;;
  version|--version) echo "rht-vmctl $VERSION" ;;
  *)                 usage; exit 1 ;;
esac
exit $RC

Host: make it executable and try it

sudo chmod 755 /usr/local/bin/rht-vmctl
bash -n /usr/local/bin/rht-vmctl && rht-vmctl version    # no syntax errors; "rht-vmctl 3"
rht-vmctl status all

Host: verify the whole build

The clean snapshot is what every reset returns to, so check everything first. This changes nothing: it prints PASS or FAIL for every requirement in this chapter, including that workstation has no project yet. Every line should pass (the utility line only matters if you built it).

P=rhce; ok(){ printf '  \e[32mPASS\e[0m %s\n' "$1"; }; ko(){ printf '  \e[31mFAIL\e[0m %s\n' "$1"; }
chk(){ if eval "$2" >/dev/null 2>&1; then ok "$1"; else ko "$1"; fi; }
chk_not(){ if eval "$2" >/dev/null 2>&1; then ko "$1"; else ok "$1"; fi; }
X(){ lxc exec --project $P "$1" -- bash -c "$2" </dev/null; }
S(){ lxc exec --project $P workstation -- su - student -c "$1" </dev/null; }
GW=$(ip route | awk '/^default/{print $3; exit}')
LANIP=$(hostname -I | awk '{print $1}')

echo "== host"
chk "LXD on 5.21/stable and held"   "snap list lxd | grep -q '5.21/stable.*held'"
chk "seal service active"           "systemctl is-active --quiet rhce-isolate"
chk "rhcebr0 up at .254"            "ip -4 addr show rhcebr0 | grep -q 172.25.250.254/24"
chk_not "host cannot ping servera"  "ping -c1 -W2 172.25.250.10"

echo "== VMs"
for vm in workstation servera serverb serverc serverd utility; do
  lxc info --project $P "$vm" >/dev/null 2>&1 || { echo "  skip $vm (not created)"; continue; }
  chk "$vm running"                 "lxc list --project $P ^$vm\$ -c s --format csv | grep -q RUNNING"
  chk "$vm FQDN hostname"           "X $vm 'hostname | grep -q ^$vm.lab.example.com\
Loading…
  #39;"
  chk "$vm profile packages"        "X $vm 'rpm -q lvm2 tar rsync vim-enhanced firewalld chrony policycoreutils-python-utils'"
  chk "$vm SELinux enforcing"       "X $vm 'test \$(getenforce) = Enforcing'"
  chk "$vm firewalld running"       "X $vm 'systemctl is-active --quiet firewalld'"
done

echo "== servers"
for vm in servera serverb serverc serverd; do
  chk "$vm sdb present and empty"    "X $vm 'test -b /dev/sdb && [ -z \"\$(lsblk -no FSTYPE /dev/sdb)\" ] && [ \$(lsblk -no NAME /dev/sdb | wc -l) -eq 1 ]'"
  chk "$vm devops passwordless sudo" "X $vm 'sudo -u devops sudo -n true'"
done

echo "== workstation"
chk "ansible is /usr/bin, core 2.14"    "S 'which -a ansible | grep -vq /usr/bin/ansible && exit 1; ansible --version | grep -q \"core 2.14\"'"
chk "ansible-navigator 26.9.0"          "S 'ansible-navigator --version | grep -q 26.9.0'"
chk "navigator EE disabled"             "S 'grep -q \"enabled: false\" ~/.ansible-navigator.yml'"
chk "collections for core 2.14"         "S 'ansible-galaxy collection list 2>/dev/null | grep -q \"^ansible.posix  *1.5\" && ansible-galaxy collection list 2>/dev/null | grep -q \"^community.general  *9\\.\"'"
chk "no ~/ansible project (clean)"      "S '! test -e ~/ansible'"
chk "SSH keys: devops, student, root"   "S 'for h in workstation servera serverb serverc serverd; do for u in devops student root; do ssh -o BatchMode=yes \$u@\$h true || exit 1; done; done'"
chk "lab command installed"             "S 'lab version'"
chk "names resolve inside lab"          "S 'getent hosts serverd.lab.example.com'"
chk "internet via NAT"                  "S 'curl -sfI -m5 https://rockylinux.org'"
chk "utility serves files (optional)"   "S 'curl -sf -m5 http://materials.example.com/files/user_list.yml'"
chk_not "lab cannot reach host LAN IP"  "S 'ping -c1 -W2 $LANIP'"
chk_not "lab cannot reach router"       "S 'ping -c1 -W2 $GW'"

Host: take the clean baseline

When everything passes:

rht-vmctl save          # snapshot 'clean' on every VM; asks before replacing an existing one

Then confirm each VM has exactly one snapshot, clean. Remove any older ones with rht-vmctl rmsnap VM NAME, so that clean is always the newest (see the ZFS rule below).

for vm in workstation servera serverb serverc serverd utility; do
  lxc info --project rhce "$vm" >/dev/null 2>&1 || continue
  n=$(lxc query "/1.0/instances/$vm/snapshots?project=rhce" | sed -n 's#.*/snapshots/\([^?"]*\).*#\1#p' | paste -sd,)
  [ "$n" = clean ] && echo "  PASS $vm: $n" || echo "  FAIL $vm: $n"
done
for vm in servera serverb serverc serverd; do          # the extra disks have their own snapshots
  n=$(lxc query "/1.0/storage-pools/default/volumes/custom/$vm-disk2/snapshots?project=rhce" | sed -n 's#.*/snapshots/\([^?"]*\).*#\1#p' | paste -sd,)
  [ "$n" = clean ] && echo "  PASS $vm-disk2: $n" || echo "  FAIL $vm-disk2: $n"
done