From 6fe0acc06159c01083d8775f2df5f2c480440212 Mon Sep 17 00:00:00 2001 From: Ruben Hensen Date: Sun, 25 Oct 2026 07:00:49 +0100 Subject: [PATCH] backup-testing: a restore test from a read-only copy of the bucket --- backup-testing/README.md | 14 +++ backup-testing/restore-test.sh | 212 +++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100755 backup-testing/restore-test.sh diff --git a/backup-testing/README.md b/backup-testing/README.md index 2fc8781..64156c1 100644 --- a/backup-testing/README.md +++ b/backup-testing/README.md @@ -1,5 +1,19 @@ # Velero Backup Testing Guide +**Use `restore-test.sh`.** It restores a few namespaces from a real backup +into a throwaway k3s in Docker, from a read-only copy of the bucket, and +checks that every volume comes back with the bytes it was backed up with: + +```bash +backup-testing/restore-test.sh # newest daily backup; ntfy, freshrss, actual-budget +backup-testing/restore-test.sh ... # something else +``` + +How it works and why it copies instead of pointing Velero at the NAS is at +the top of the script. The rest of this README describes the older k3d +scripts, which predate Envoy Gateway and still expect `*.hensen.io` and +sol-auth. + This guide explains how to test your Velero backups using a temporary K3d cluster. ## Features diff --git a/backup-testing/restore-test.sh b/backup-testing/restore-test.sh new file mode 100755 index 0000000..1bc741b --- /dev/null +++ b/backup-testing/restore-test.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# Restore a few namespaces from the real Velero backups into a throwaway +# k3s in Docker, to prove the backups can be restored. Nothing on the NAS +# changes: +# - one backup (backups//) and the kopia repositories of the chosen +# namespaces (kopia//) are copied read-only from MinIO with +# rclone into a Docker volume. A second Velero on the real bucket could +# run kopia maintenance on the same repositories as the cluster's own +# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a +# ReadOnly backup storage location +# - storage class longhorn becomes local-path; only core workload +# resources are restored (no routes, certificates, ExternalSecrets or +# NetworkPolicies) +# - every pod volume restore has to complete with as many bytes as its +# backup had, and the restored deployments have to become available +# +# Usage (on a machine with docker, kubectl and jq): +# backup-testing/restore-test.sh [backup] [namespace ...] +# Defaults: the newest Completed daily-backup, and ntfy freshrss +# actual-budget. Not gatus: a restored Gatus would start sending alerts. +# +# It reads Velero's MinIO key from the cluster (velero/cloud-credentials) +# into a temporary directory, and the copied backup holds every Secret of +# the cluster; all of it is deleted at the end. KEEP=1 keeps the test +# cluster (kubeconfig in the printed directory) until you run it with +# CLEANUP=1. FROM_VOLUME= takes the bucket from a volume +# instead of the NAS (for rehearsals; laid out as /backups/...). +# +# Rehearsed on 2026-10-25 against a backup made in the test cluster itself, +# which is how two pitfalls showed up: Velero skips hostPath volumes, and +# k3s only provisions local PVs (which it can back up and restore) for a +# storage class with the annotation defaultVolumeType: local. +set -euo pipefail + +PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config} +MINIO=${MINIO:-http://192.168.1.99:9000} +BUCKET=velero-backups +PORT=${PORT:-16443} +K3S=rancher/k3s:v1.34.9-k3s1 +RCLONE=rclone/rclone:1.75.1 +HELM=alpine/helm:3.19.0 +VELERO_CHART=12.2.1 +AWS_PLUGIN=velero/velero-plugin-for-aws:v1.14.4 +NET=restore-test VOL=restore-test-s3 + +cleanup() { + docker rm -f restore-k3s restore-s3 >/dev/null 2>&1 || true + docker volume rm "$VOL" >/dev/null 2>&1 || true + docker network rm "$NET" >/dev/null 2>&1 || true + [[ -n ${WORK:-} ]] && rm -rf "$WORK" +} +if [[ ${CLEANUP:-} == 1 ]]; then + cleanup; rm -rf "${TMPDIR:-/tmp}"/restore-test.*; echo "cleaned up"; exit 0 +fi +if docker ps -a --format '{{.Names}}' | grep -qx restore-k3s; then + echo "A test cluster from an earlier run exists; run with CLEANUP=1 first." >&2 + exit 1 +fi + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/restore-test.XXXXXX") +chmod 700 "$WORK" +[[ ${KEEP:-} == 1 ]] || trap cleanup EXIT + +pk() { kubectl --kubeconfig "$PROD_KUBECONFIG" "$@"; } +tk() { kubectl --kubeconfig "$WORK/kubeconfig" "$@"; } +# wait_for +wait_for() { + local limit=$1 what=$2; shift 2 + local end=$((SECONDS + limit)) + until "$@" >/dev/null 2>&1; do + if ((SECONDS > end)); then echo "timed out waiting for $what" >&2; exit 1; fi + sleep 5 + done +} + +backup=${1:-} +shift || true +namespaces=("$@") +[[ ${#namespaces[@]} -gt 0 ]] || namespaces=(ntfy freshrss actual-budget) +if [[ -z $backup ]]; then + backup=$(pk -n velero get backups.velero.io -o json | jq -r '[.items[] + | select(.metadata.labels["velero.io/schedule-name"] == "daily-backup" and .status.phase == "Completed")] + | max_by(.status.startTimestamp).metadata.name') +fi +echo "== backup $backup, namespaces: ${namespaces[*]}" + +docker network create "$NET" >/dev/null +docker volume create "$VOL" >/dev/null +if [[ -n ${FROM_VOLUME:-} ]]; then + echo "== copy from the volume $FROM_VOLUME" + docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/ +else +echo "== copy from the NAS (read-only)" +(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d | + awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2} + /aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env" + printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env") +copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; } +copy "backups/$backup" +for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done +rm -f "$WORK/nas.env" +fi + +echo "== local S3 and k3s" +docker run -d --name restore-s3 --network "$NET" -v "$VOL:/data" "$RCLONE" \ + serve s3 /data --addr :8080 --auth-key test,restore-test-secret >/dev/null +s3ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" restore-s3) +docker run -d --name restore-k3s --privileged --network "$NET" -p "127.0.0.1:$PORT:6443" "$K3S" \ + server --disable traefik --disable metrics-server --tls-san 127.0.0.1 --tls-san restore-k3s >/dev/null +node_ready() { docker exec restore-k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; } +wait_for 180 "k3s" node_ready +docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$PORT#" > "$WORK/kubeconfig" +docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed 's#https://127.0.0.1:6443#https://restore-k3s:6443#' > "$WORK/kubeconfig-net" + +echo "== Velero" +cat > "$WORK/values.yaml" < /tmp/kc && echo "$VALUES" | base64 -d > /tmp/v.yaml && + helm --kubeconfig /tmp/kc install velero velero --repo https://vmware-tanzu.github.io/helm-charts \ + --version '"$VELERO_CHART"' -n velero --create-namespace -f /tmp/v.yaml >/dev/null' +tk apply -f - >/dev/null </dev/null </dev/null 2>&1; then + echo " ok $ns: deployments available" + else + echo " FAILED $ns: deployments not available"; tk -n "$ns" get pods; failed=1 + fi +done +[[ $phase == Completed ]] || failed=1 +[[ ${KEEP:-} == 1 ]] && echo "Kept; kubeconfig: $WORK/kubeconfig. Remove with CLEANUP=1 $0" +if [[ $failed == 0 ]]; then echo "== restore test passed"; else echo "== restore test FAILED"; exit 1; fi