From 4607163d4e7d33d377f43b4798eeb3704c475d84 Mon Sep 17 00:00:00 2001
From: Ruben Hensen <ruben.hensen@protonmail.com>
Date: Mon, 3 Mar 2025 22:40:24 +0100
Subject: [PATCH] Change schedule backup

---
 .gitignore                                    |   3 +-
 backup-testing/readme.md                      | 139 +++++++
 backup-testing/setup-credentials.sh           |  39 ++
 backup-testing/test-velero-backup.sh          | 371 ++++++++++++++++++
 .../restore-backup}/README.md                 |   0
 .../restore-backup}/restore-ubuntu.yaml       |   0
 longhorn/volume-snapshot.yaml                 |   2 +-
 velero/backup-schedule.yaml                   |   6 +-
 8 files changed, 557 insertions(+), 3 deletions(-)
 create mode 100644 backup-testing/readme.md
 create mode 100755 backup-testing/setup-credentials.sh
 create mode 100755 backup-testing/test-velero-backup.sh
 rename {restore-backup => disable/restore-backup}/README.md (100%)
 rename {restore-backup => disable/restore-backup}/restore-ubuntu.yaml (100%)

diff --git a/.gitignore b/.gitignore
index 642b1ff..23b2616 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 ente-photos-ref/
 repomix-output.txt
-credentials-velero
\ No newline at end of file
+credentials-velero
+velero-credentials
\ No newline at end of file
diff --git a/backup-testing/readme.md b/backup-testing/readme.md
new file mode 100644
index 0000000..a0b0652
--- /dev/null
+++ b/backup-testing/readme.md
@@ -0,0 +1,139 @@
+# Velero Backup Testing Guide
+
+This guide explains how to test your Velero backups using a temporary K3d cluster.
+
+## Prerequisites
+
+Ensure you have the following tools installed on your local machine:
+
+- Docker
+- kubectl
+- Helm
+- K3d
+- Velero CLI
+
+## Setup
+
+1. First, clone this repository to your local machine:
+
+```bash
+git clone https://github.com/yourusername/backup-testing.git
+cd backup-testing
+```
+
+2. Make the scripts executable:
+
+```bash
+chmod +x setup-credentials.sh test-velero-backup.sh validate-restore.sh
+```
+
+3. Set up your MinIO credentials:
+
+```bash
+./setup-credentials.sh
+```
+
+This script will look for credentials in `~/.velero/credentials-velero` or prompt you to create them.
+
+## Running a Backup Test
+
+Execute the test script with appropriate parameters:
+
+```bash
+./test-velero-backup.sh \
+  --s3-access-key YOUR_ACCESS_KEY \
+  --s3-secret-key YOUR_SECRET_KEY \
+  --original-namespace immich
+```
+
+This will:
+1. Create a temporary K3d cluster
+2. Install Longhorn for storage
+3. Install Velero configured to access your MinIO backup location
+4. List available backups
+5. Allow you to test restoring a specific backup
+
+To restore a specific backup, run:
+
+```bash
+./test-velero-backup.sh \
+  --s3-access-key YOUR_ACCESS_KEY \
+  --s3-secret-key YOUR_SECRET_KEY \
+  --original-namespace immich \
+  --backup-name specific-backup-name
+```
+
+## Testing Specific Applications
+
+### Immich
+
+For testing Immich backups:
+
+```bash
+./test-velero-backup.sh \
+  --s3-access-key YOUR_ACCESS_KEY \
+  --s3-secret-key YOUR_SECRET_KEY \
+  --original-namespace immich \
+  --test-namespace immich-test
+```
+
+### NextCloud
+
+For testing NextCloud backups:
+
+```bash
+./test-velero-backup.sh \
+  --s3-access-key YOUR_ACCESS_KEY \
+  --s3-secret-key YOUR_SECRET_KEY \
+  --original-namespace nextcloud \
+  --test-namespace nextcloud-test
+```
+
+## Testing with Volume Snapshots
+
+To test CSI volume snapshot backups:
+
+```bash
+./test-velero-backup.sh \
+  --s3-access-key YOUR_ACCESS_KEY \
+  --s3-secret-key YOUR_SECRET_KEY \
+  --original-namespace immich \
+  --include-volume-snapshots
+```
+
+## Customizing Validation
+
+You can extend the `validate-restore.sh` script to perform application-specific validation checks:
+
+1. Edit the script to add application-specific checks
+2. Run your test with the modified validation:
+
+```bash
+./test-velero-backup.sh \
+  --s3-access-key YOUR_ACCESS_KEY \
+  --s3-secret-key YOUR_SECRET_KEY \
+  --original-namespace immich \
+  --backup-name your-backup \
+  --custom-validation-script ./your-custom-validation.sh
+```
+
+## Troubleshooting
+
+If you encounter issues:
+
+1. Check the logs of the Velero pod:
+   ```
+   kubectl logs -n velero deploy/velero
+   ```
+
+2. Examine restore details:
+   ```
+   velero restore describe RESTORE_NAME
+   velero restore logs RESTORE_NAME
+   ```
+
+3. Check cluster resources:
+   ```
+   kubectl get pods -A
+   kubectl get pvc -A
+   ```
diff --git a/backup-testing/setup-credentials.sh b/backup-testing/setup-credentials.sh
new file mode 100755
index 0000000..b77cc96
--- /dev/null
+++ b/backup-testing/setup-credentials.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# setup-credentials.sh - Script to help setup credentials for Velero testing
+
+# Directory where Velero credentials are stored on the production system
+# This would be your production system's credentials for accessing MinIO
+VELERO_CREDS_PATH="${HOME}/.velero/credentials-velero"
+
+if [ -f "${VELERO_CREDS_PATH}" ]; then
+  echo "Found Velero credentials at ${VELERO_CREDS_PATH}"
+  # Extract AWS credentials from the file
+  AWS_ACCESS_KEY_ID=$(grep aws_access_key_id ${VELERO_CREDS_PATH} | cut -d= -f2 | tr -d '[:space:]')
+  AWS_SECRET_ACCESS_KEY=$(grep aws_secret_access_key ${VELERO_CREDS_PATH} | cut -d= -f2 | tr -d '[:space:]')
+  
+  if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then
+    echo "Credentials found and will be used for backup testing"
+    
+    # Store them temporarily for the test script to use
+    cat > /tmp/velero-test-credentials <<EOF
+[default]
+aws_access_key_id=${AWS_ACCESS_KEY_ID}
+aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
+EOF
+    
+    echo "Credentials saved to /tmp/velero-test-credentials"
+    echo ""
+    echo "Run your test script with:"
+    echo "./test-velero-backup.sh --s3-access-key ${AWS_ACCESS_KEY_ID} --s3-secret-key ${AWS_SECRET_ACCESS_KEY} [other options]"
+  else
+    echo "Error: Could not extract credentials from ${VELERO_CREDS_PATH}"
+    exit 1
+  fi
+else
+  echo "Error: Velero credentials file not found at ${VELERO_CREDS_PATH}"
+  echo "Please create this file with the following format:"
+  echo "[default]"
+  echo "aws_access_key_id=YOUR_MINIO_ACCESS_KEY"
+  echo "aws_secret_access_key=YOUR_MINIO_SECRET_KEY"
+  exit 1
+fi
diff --git a/backup-testing/test-velero-backup.sh b/backup-testing/test-velero-backup.sh
new file mode 100755
index 0000000..dbe8289
--- /dev/null
+++ b/backup-testing/test-velero-backup.sh
@@ -0,0 +1,371 @@
+#!/bin/bash
+# test-velero-backup.sh - Automated script to test Velero backups
+
+set -e
+
+# Configuration - customize these variables
+BACKUP_NAME=""          # Will list backups if empty
+S3_BUCKET="velero-backups"
+S3_REGION="minio"
+S3_URL="http://192.168.1.99:9000"               # Your NAS MinIO server
+S3_ACCESS_KEY=""  # Will be filled from credentials
+S3_SECRET_KEY=""  # Will be filled from credentials
+ORIGINAL_NAMESPACE=""   # Namespace in the original backup
+TEST_NAMESPACE="default"
+DEBUG=false             # Set to true for verbose debugging
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --backup-name)
+      BACKUP_NAME="$2"
+      shift 2
+      ;;
+    --s3-access-key)
+      S3_ACCESS_KEY="$2"
+      shift 2
+      ;;
+    --s3-secret-key)
+      S3_SECRET_KEY="$2"
+      shift 2
+      ;;
+    --s3-bucket)
+      S3_BUCKET="$2"
+      shift 2
+      ;;
+    --s3-region)
+      S3_REGION="$2"
+      shift 2
+      ;;
+    --s3-url)
+      S3_URL="$2"
+      shift 2
+      ;;
+    --original-namespace)
+      ORIGINAL_NAMESPACE="$2"
+      shift 2
+      ;;
+    --test-namespace)
+      TEST_NAMESPACE="$2"
+      shift 2
+      ;;
+    --debug)
+      DEBUG=true
+      shift
+      ;;
+    --help)
+      echo "Usage: $0 [options]"
+      echo "Options:"
+      echo "  --backup-name NAME         Name of the Velero backup to restore"
+      echo "  --s3-bucket BUCKET         S3 bucket containing backups"
+      echo "  --s3-region REGION         S3 region"
+      echo "  --s3-url URL               S3 endpoint URL"
+      echo "  --s3-access-key KEY        S3 access key"
+      echo "  --s3-secret-key KEY        S3 secret key"
+      echo "  --original-namespace NS    Namespace in the original backup"
+      echo "  --test-namespace NS        Namespace for restoration (default: default)"
+      echo "  --debug                    Enable debug output"
+      echo "  --help                     Show this help message"
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+# Verify required variables
+if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" ]]; then
+  echo "Error: S3 configuration is incomplete"
+  exit 1
+fi
+
+# Function to clean up resources
+cleanup() {
+  echo "Cleaning up resources..."
+  k3d cluster delete backup-test || true
+  echo "Cleanup complete"
+}
+
+# Function for debugging output
+debug() {
+  if [[ "$DEBUG" == "true" ]]; then
+    echo "[DEBUG] $*"
+  fi
+}
+
+# Register cleanup function to run on exit (ctrl+c, etc.)
+trap cleanup EXIT
+
+echo "=== Starting Velero backup test process ==="
+
+# 1. Create K3d cluster
+echo "Creating K3d cluster..."
+k3d cluster create backup-test \
+  --api-port 6443 \
+  --port "80:80@loadbalancer" \
+  --port "443:443@loadbalancer" \
+  --agents 2 \
+  --k3s-arg "--disable=traefik@server:0" \
+  --wait
+
+# Verify cluster is running
+kubectl config use-context k3d-backup-test
+kubectl get nodes
+
+# 2. Install Longhorn
+echo "Installing Longhorn..."
+kubectl create namespace longhorn-system
+
+helm repo add longhorn https://charts.longhorn.io
+helm repo update
+
+helm install longhorn longhorn/longhorn \
+  --namespace longhorn-system \
+  --set persistence.defaultClassReplicaCount=1 \
+  --set defaultSettings.backupTarget="" \
+  --set defaultSettings.defaultReplicaCount=1
+
+echo "Waiting for Longhorn to be ready..."
+kubectl -n longhorn-system rollout status deployment/longhorn-ui
+
+# 3. Install Velero with S3 provider
+echo "Installing Velero..."
+cat > velero-credentials <<EOF
+[default]
+aws_access_key_id=$S3_ACCESS_KEY
+aws_secret_access_key=$S3_SECRET_KEY
+EOF
+
+helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
+helm repo update
+
+debug "Installing Velero with S3 configuration:"
+debug "  Bucket: $S3_BUCKET"
+debug "  Region: $S3_REGION"
+debug "  URL: $S3_URL"
+
+helm install velero vmware-tanzu/velero \
+  --namespace velero \
+  --create-namespace \
+  --set-file credentials.secretContents.cloud=velero-credentials \
+  --set configuration.backupStorageLocation[0].provider=aws \
+  --set configuration.backupStorageLocation[0].name=default \
+  --set configuration.backupStorageLocation[0].bucket=$S3_BUCKET \
+  --set configuration.backupStorageLocation[0].config.region=$S3_REGION \
+  --set configuration.backupStorageLocation[0].config.s3ForcePathStyle=true \
+  --set configuration.backupStorageLocation[0].config.s3Url=$S3_URL \
+  --set configuration.volumeSnapshotLocation[0].name=default \
+  --set configuration.volumeSnapshotLocation[0].provider=aws \
+  --set configuration.volumeSnapshotLocation[0].config.region=$S3_REGION \
+  --set configuration.features=EnableCSI \
+  --set initContainers[0].name=velero-plugin-for-aws \
+  --set initContainers[0].image=velero/velero-plugin-for-aws:v1.11.1 \
+  --set initContainers[0].volumeMounts[0].mountPath=/target \
+  --set initContainers[0].volumeMounts[0].name=plugins \
+  --set deployNodeAgent=true \
+  --set nodeAgent.containerSecurityContext.privileged=true \
+  --wait
+
+rm velero-credentials
+
+# Wait for Velero to be ready
+echo "Waiting for Velero to be fully ready..."
+kubectl -n velero rollout status deployment/velero --timeout=120s
+
+# Check if velero CLI is installed and functioning
+if ! command -v velero &> /dev/null; then
+  echo "Error: Velero CLI is not installed or not in PATH"
+  echo "Please install the Velero CLI by following instructions at: https://velero.io/docs/main/basic-install/"
+  exit 1
+fi
+
+# Verify Velero is connected to the storage location
+echo "Checking Velero backup storage location..."
+kubectl -n velero get backupstoragelocation default -o jsonpath='{.status.phase}' | grep -q "Available" || {
+  echo "Warning: Backup storage location is not available, it may take a few minutes to initialize"
+  echo "Storage location status:"
+  kubectl -n velero get backupstoragelocation default -o yaml
+  
+  # Wait a bit longer for the location to become available
+  echo "Waiting 30 seconds for backup storage location to become available..."
+  sleep 30
+}
+
+# 4. List backups
+echo "Checking for available backups..."
+if ! velero_output=$(velero backup get 2>&1); then
+  echo "Error listing backups: $velero_output"
+  
+  # Try to verify S3 connectivity
+  echo "Checking S3 connectivity..."
+  
+  # Install AWS CLI if needed
+  if ! command -v aws &> /dev/null; then
+    echo "AWS CLI not found, skipping S3 connectivity test"
+  else
+    # Create temporary AWS profile
+    mkdir -p ~/.aws
+    cat > ~/.aws/credentials <<EOF
+[velero-test]
+aws_access_key_id=$S3_ACCESS_KEY
+aws_secret_access_key=$S3_SECRET_KEY
+EOF
+
+    cat > ~/.aws/config <<EOF
+[profile velero-test]
+region=$S3_REGION
+output=json
+EOF
+
+    # Try to list bucket contents
+    echo "Testing S3 connection to bucket $S3_BUCKET..."
+    if aws --profile velero-test --endpoint-url $S3_URL s3 ls s3://$S3_BUCKET/; then
+      echo "S3 connection successful, but Velero couldn't list backups."
+      echo "This might indicate that there are no backups in the bucket or the backups are in a different format than expected."
+    else
+      echo "Error connecting to S3. Please check your credentials and connectivity to $S3_URL"
+    fi
+    
+    # Cleanup AWS files
+    rm ~/.aws/credentials ~/.aws/config
+  fi
+  
+  if [[ -z "$BACKUP_NAME" ]]; then
+    echo "No backup name specified and no backups found. Exiting."
+    exit 1
+  fi
+else
+  echo "Available backups:"
+  echo "$velero_output"
+  
+  if [[ -z "$BACKUP_NAME" ]]; then
+    echo "Specify a backup name using --backup-name and run again."
+    exit 0
+  fi
+fi
+
+# 5. Create test namespace if needed
+if [[ "$TEST_NAMESPACE" != "default" ]]; then
+  kubectl create namespace $TEST_NAMESPACE || true
+fi
+
+# 6. Restore backup
+echo "Restoring backup: $BACKUP_NAME"
+if [[ -z "$ORIGINAL_NAMESPACE" ]]; then
+  # If no namespace specified, restore everything
+  velero restore create --from-backup $BACKUP_NAME \
+    --wait
+else
+  # Restore specific namespace with mapping
+  velero restore create --from-backup $BACKUP_NAME \
+    --namespace-mappings $ORIGINAL_NAMESPACE:$TEST_NAMESPACE \
+    --include-namespaces $ORIGINAL_NAMESPACE \
+    --wait
+fi
+
+# 7. Check restore status
+echo "Checking restore status:"
+velero restore get
+
+# Wait for pods to be ready
+echo "Waiting for pods to be ready..."
+kubectl wait --for=condition=ready pod --all -n $TEST_NAMESPACE --timeout=300s || true
+
+# 8. Run validation script (create it first if it doesn't exist)
+if [[ ! -f ./validate-restore.sh ]]; then
+  echo "Creating validation script..."
+  cat > validate-restore.sh <<'EOF'
+#!/bin/bash
+# validate-restore.sh - Script to validate Velero backup restoration
+
+set -e
+echo "Starting validation of restored resources..."
+
+# Get namespace from command line or use default
+NAMESPACE="${1:-default}"
+
+# Check all deployments
+echo "Checking deployments..."
+deployments=$(kubectl get deployment -n $NAMESPACE -o name)
+if [[ -z "$deployments" ]]; then
+  echo "❌ No deployments found in namespace $NAMESPACE"
+else
+  echo "Found deployments: $deployments"
+  
+  # Check if pods are running for each deployment
+  for deployment in $deployments; do
+    name=$(echo $deployment | cut -d'/' -f2)
+    echo -n "Checking deployment $name: "
+    
+    # Check if pods are running
+    ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
+    if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
+      echo "✅ $ready_replicas pods ready"
+    else
+      echo "❌ No pods running"
+    fi
+  done
+fi
+
+# Check all services
+echo "Checking services..."
+services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
+if [[ -z "$services" ]]; then
+  echo "❌ No services found in namespace $NAMESPACE"
+else
+  echo "Found services: $services"
+  
+  # Check if services have endpoints
+  for service in $services; do
+    name=$(echo $service | cut -d'/' -f2)
+    echo -n "Checking service $name: "
+    
+    # Check if service has endpoints
+    endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
+    if [ -n "$endpoints" ]; then
+      echo "✅ Has endpoints"
+    else
+      echo "❌ No endpoints"
+    fi
+  done
+fi
+
+# Check PVCs
+echo "Checking PVCs..."
+pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
+if [[ -z "$pvcs" ]]; then
+  echo "No PVCs found in namespace $NAMESPACE"
+else
+  pvc_count=$(echo "$pvcs" | wc -l)
+  echo "Found $pvc_count PVCs"
+  
+  # Check if PVCs are bound
+  bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
+  echo "$bound_count/$pvc_count PVCs are bound"
+  
+  if [ "$bound_count" -ne "$pvc_count" ]; then
+    echo "❌ Not all PVCs are bound"
+  else
+    echo "✅ All PVCs are bound"
+  fi
+fi
+
+echo "Validation complete!"
+exit 0
+EOF
+  chmod +x validate-restore.sh
+fi
+
+echo "Running validation tests..."
+./validate-restore.sh $TEST_NAMESPACE
+
+echo ""
+echo "=== Backup test process completed ==="
+echo "The cluster will be deleted when you close this terminal or press Ctrl+C"
+echo "To access the cluster manually, use: kubectl config use-context k3d-backup-test"
+echo ""
+
+# Keep the script running until user terminates it
+read -p "Press Enter to clean up and exit..."
\ No newline at end of file
diff --git a/restore-backup/README.md b/disable/restore-backup/README.md
similarity index 100%
rename from restore-backup/README.md
rename to disable/restore-backup/README.md
diff --git a/restore-backup/restore-ubuntu.yaml b/disable/restore-backup/restore-ubuntu.yaml
similarity index 100%
rename from restore-backup/restore-ubuntu.yaml
rename to disable/restore-backup/restore-ubuntu.yaml
diff --git a/longhorn/volume-snapshot.yaml b/longhorn/volume-snapshot.yaml
index 809743d..2953df1 100644
--- a/longhorn/volume-snapshot.yaml
+++ b/longhorn/volume-snapshot.yaml
@@ -1,5 +1,5 @@
-kind: VolumeSnapshotClass
 apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
 metadata:
   name: longhorn-backup-vsc
   labels:
diff --git a/velero/backup-schedule.yaml b/velero/backup-schedule.yaml
index 393ba80..538c10f 100644
--- a/velero/backup-schedule.yaml
+++ b/velero/backup-schedule.yaml
@@ -3,8 +3,10 @@ kind: Schedule
 metadata:
   name: daily-backup
   namespace: velero
+  annotations:
+    velero.io/csi-volumesnapshot-class_driver.longhorn.io: "longhorn-backup-vsc"
 spec:
-  schedule: "0 1 * * *"  # Daily at 1 AM
+  schedule: "41 22 * * *"  # Daily at 1 AM
   template:
     # includedNamespaces:
     #   - immich
@@ -19,6 +21,8 @@ kind: Schedule
 metadata:
   name: weekly-backup
   namespace: velero
+  annotations:
+    velero.io/csi-volumesnapshot-class_driver.longhorn.io: "longhorn-backup-vsc"
 spec:
   schedule: "0 0 * * 0"  # Weekly on Sunday at midnight
   template: