Add backup testing scripts

backup-testing/README.md

@@ -39,6 +39,15 @@ This script will look for credentials in `~/.velero/credentials-velero` or promp
 
 Execute the test script with appropriate parameters:
 
+```bash
+./test-velero-backup.sh \
+--s3-access-key <ACCESS_KEY> \
+--s3-secret-key <SECRET_KEY> \
+--backup-name <BACKUP_NAME> \
+--original-namespace mbgwp \
+--test-namespace mbgwp
+```
+
 ```bash
 ./test-velero-backup.sh \
   --s3-access-key YOUR_ACCESS_KEY \

backup-testing/change-storage-class.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  # any name can be used; Velero uses the labels (below)
+  # to identify it rather than the name
+  name: change-storage-class-config
+  # must be in the velero namespace
+  namespace: velero
+  # the below labels should be used verbatim in your
+  # ConfigMap.
+  labels:
+    # this value-less label identifies the ConfigMap as
+    # config for a plugin (i.e. the built-in change storage
+    # class restore item action plugin)
+    velero.io/plugin-config: ""
+    # this label identifies the name and kind of plugin
+    # that this ConfigMap is for.
+    velero.io/change-storage-class: RestoreItemAction
+data:
+  # add 1+ key-value pairs here, where the key is the old
+  # storage class name and the value is the new storage
+  # class name.
+  longhorn: local-path

backup-testing/setup-credentials.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-# setup-credentials.sh - Script to help setup credentials for Velero testing
-
-# Directory where Velero credentials are stored on the production system
-# This would be your production system's credentials for accessing MinIO
-VELERO_CREDS_PATH="${HOME}/.velero/credentials-velero"
-
-if [ -f "${VELERO_CREDS_PATH}" ]; then
-  echo "Found Velero credentials at ${VELERO_CREDS_PATH}"
-  # Extract AWS credentials from the file
-  AWS_ACCESS_KEY_ID=$(grep aws_access_key_id ${VELERO_CREDS_PATH} | cut -d= -f2 | tr -d '[:space:]')
-  AWS_SECRET_ACCESS_KEY=$(grep aws_secret_access_key ${VELERO_CREDS_PATH} | cut -d= -f2 | tr -d '[:space:]')
-  
-  if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then
-    echo "Credentials found and will be used for backup testing"
-    
-    # Store them temporarily for the test script to use
-    cat > /tmp/velero-test-credentials <<EOF
-[default]
-aws_access_key_id=${AWS_ACCESS_KEY_ID}
-aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
-EOF
-    
-    echo "Credentials saved to /tmp/velero-test-credentials"
-    echo ""
-    echo "Run your test script with:"
-    echo "./test-velero-backup.sh --s3-access-key ${AWS_ACCESS_KEY_ID} --s3-secret-key ${AWS_SECRET_ACCESS_KEY} [other options]"
-  else
-    echo "Error: Could not extract credentials from ${VELERO_CREDS_PATH}"
-    exit 1
-  fi
-else
-  echo "Error: Velero credentials file not found at ${VELERO_CREDS_PATH}"
-  echo "Please create this file with the following format:"
-  echo "[default]"
-  echo "aws_access_key_id=YOUR_MINIO_ACCESS_KEY"
-  echo "aws_secret_access_key=YOUR_MINIO_SECRET_KEY"
-  exit 1
-fi

backup-testing/test-velero-backup.sh

@@ -76,7 +76,7 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Verify required variables
-if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" ]]; then
+if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" || -z "$ORIGINAL_NAMESPACE" || -z "$TEST_NAMESPACE" ]]; then
   echo "Error: S3 configuration is incomplete"
   exit 1
 fi
@@ -106,29 +106,34 @@ k3d cluster create backup-test \
   --api-port 6443 \
   --port "80:80@loadbalancer" \
   --port "443:443@loadbalancer" \
-  --agents 2 \
+  --agents 0 \
   --k3s-arg "--disable=traefik@server:0" \
   --wait
+  # -v /mnt/longhorn:/var/lib/longhorn:shared@all \
+  # -i hebury/k3s:v1.32.2-k3s1  \
+  # --wait
 
 # Verify cluster is running
 kubectl config use-context k3d-backup-test
 kubectl get nodes
 
 # 2. Install Longhorn
-echo "Installing Longhorn..."
+# echo "Installing Longhorn..."
-kubectl create namespace longhorn-system
+# kubectl create namespace longhorn-system
 
-helm repo add longhorn https://charts.longhorn.io
+# helm repo add longhorn https://charts.longhorn.io
-helm repo update
+# helm repo update
 
-helm install longhorn longhorn/longhorn \
+# helm install longhorn longhorn/longhorn \
-  --namespace longhorn-system \
+#   --namespace longhorn-system \
-  --set persistence.defaultClassReplicaCount=1 \
+#   --set persistence.defaultClassReplicaCount=1 \
-  --set defaultSettings.backupTarget="" \
+#   --set defaultSettings.backupTarget="" \
-  --set defaultSettings.defaultReplicaCount=1
+#   --set defaultSettings.defaultReplicaCount=1
 
-echo "Waiting for Longhorn to be ready..."
+# echo "Waiting for Longhorn to be ready..."
-kubectl -n longhorn-system rollout status deployment/longhorn-ui
+# kubectl -n longhorn-system rollout status deployment/longhorn-ui
+# kubectl -n longhorn-system rollout status deployment/longhorn-driver-deployer
+# kubectl -n longhorn-system rollout status daemonset/longhorn-manager
 
 # 3. Install Velero with S3 provider
 echo "Installing Velero..."
@@ -174,6 +179,9 @@ rm velero-credentials
 echo "Waiting for Velero to be fully ready..."
 kubectl -n velero rollout status deployment/velero --timeout=120s
 
+echo "Install change-storage-class.yaml"
+kubectl apply -f change-storage-class.yaml
+
 # Check if velero CLI is installed and functioning
 if ! command -v velero &> /dev/null; then
   echo "Error: Velero CLI is not installed or not in PATH"
@@ -262,6 +270,8 @@ else
   velero restore create --from-backup $BACKUP_NAME \
     --namespace-mappings $ORIGINAL_NAMESPACE:$TEST_NAMESPACE \
     --include-namespaces $ORIGINAL_NAMESPACE \
+    --exclude-namespaces kube-system,kube-public,kube-node-lease,velero \
+    --exclude-resources certificates.cert-manager.io \ 
     --wait
 fi
 
@@ -274,89 +284,89 @@ echo "Waiting for pods to be ready..."
 kubectl wait --for=condition=ready pod --all -n $TEST_NAMESPACE --timeout=300s || true
 
 # 8. Run validation script (create it first if it doesn't exist)
-if [[ ! -f ./validate-restore.sh ]]; then
+# if [[ ! -f ./validate-restore.sh ]]; then
-  echo "Creating validation script..."
+#   echo "Creating validation script..."
-  cat > validate-restore.sh <<'EOF'
+#   cat > validate-restore.sh <<'EOF'
-#!/bin/bash
+# #!/bin/bash
-# validate-restore.sh - Script to validate Velero backup restoration
+# # validate-restore.sh - Script to validate Velero backup restoration
 
-set -e
+# set -e
-echo "Starting validation of restored resources..."
+# echo "Starting validation of restored resources..."
 
-# Get namespace from command line or use default
+# # Get namespace from command line or use default
-NAMESPACE="${1:-default}"
+# NAMESPACE="${1:-default}"
 
-# Check all deployments
+# # Check all deployments
-echo "Checking deployments..."
+# echo "Checking deployments..."
-deployments=$(kubectl get deployment -n $NAMESPACE -o name)
+# deployments=$(kubectl get deployment -n $NAMESPACE -o name)
-if [[ -z "$deployments" ]]; then
+# if [[ -z "$deployments" ]]; then
-  echo "❌ No deployments found in namespace $NAMESPACE"
+#   echo "❌ No deployments found in namespace $NAMESPACE"
-else
+# else
-  echo "Found deployments: $deployments"
+#   echo "Found deployments: $deployments"
   
-  # Check if pods are running for each deployment
+#   # Check if pods are running for each deployment
-  for deployment in $deployments; do
+#   for deployment in $deployments; do
-    name=$(echo $deployment | cut -d'/' -f2)
+#     name=$(echo $deployment | cut -d'/' -f2)
-    echo -n "Checking deployment $name: "
+#     echo -n "Checking deployment $name: "
     
-    # Check if pods are running
+#     # Check if pods are running
-    ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
+#     ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
-    if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
+#     if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
-      echo "✅ $ready_replicas pods ready"
+#       echo "✅ $ready_replicas pods ready"
-    else
+#     else
-      echo "❌ No pods running"
+#       echo "❌ No pods running"
-    fi
+#     fi
-  done
+#   done
-fi
+# fi
 
-# Check all services
+# # Check all services
-echo "Checking services..."
+# echo "Checking services..."
-services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
+# services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
-if [[ -z "$services" ]]; then
+# if [[ -z "$services" ]]; then
-  echo "❌ No services found in namespace $NAMESPACE"
+#   echo "❌ No services found in namespace $NAMESPACE"
-else
+# else
-  echo "Found services: $services"
+#   echo "Found services: $services"
   
-  # Check if services have endpoints
+#   # Check if services have endpoints
-  for service in $services; do
+#   for service in $services; do
-    name=$(echo $service | cut -d'/' -f2)
+#     name=$(echo $service | cut -d'/' -f2)
-    echo -n "Checking service $name: "
+#     echo -n "Checking service $name: "
     
-    # Check if service has endpoints
+#     # Check if service has endpoints
-    endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
+#     endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
-    if [ -n "$endpoints" ]; then
+#     if [ -n "$endpoints" ]; then
-      echo "✅ Has endpoints"
+#       echo "✅ Has endpoints"
-    else
+#     else
-      echo "❌ No endpoints"
+#       echo "❌ No endpoints"
-    fi
+#     fi
-  done
+#   done
-fi
+# fi
 
-# Check PVCs
+# # Check PVCs
-echo "Checking PVCs..."
+# echo "Checking PVCs..."
-pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
+# pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
-if [[ -z "$pvcs" ]]; then
+# if [[ -z "$pvcs" ]]; then
-  echo "No PVCs found in namespace $NAMESPACE"
+#   echo "No PVCs found in namespace $NAMESPACE"
-else
+# else
-  pvc_count=$(echo "$pvcs" | wc -l)
+#   pvc_count=$(echo "$pvcs" | wc -l)
-  echo "Found $pvc_count PVCs"
+#   echo "Found $pvc_count PVCs"
   
-  # Check if PVCs are bound
+#   # Check if PVCs are bound
-  bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
+#   bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
-  echo "$bound_count/$pvc_count PVCs are bound"
+#   echo "$bound_count/$pvc_count PVCs are bound"
   
-  if [ "$bound_count" -ne "$pvc_count" ]; then
+#   if [ "$bound_count" -ne "$pvc_count" ]; then
-    echo "❌ Not all PVCs are bound"
+#     echo "❌ Not all PVCs are bound"
-  else
+#   else
-    echo "✅ All PVCs are bound"
+#     echo "✅ All PVCs are bound"
-  fi
+#   fi
-fi
+# fi
 
-echo "Validation complete!"
+# echo "Validation complete!"
-exit 0
+# exit 0
-EOF
+# EOF
-  chmod +x validate-restore.sh
+#   chmod +x validate-restore.sh
-fi
+# fi
 
 echo "Running validation tests..."
 ./validate-restore.sh $TEST_NAMESPACE

backup-testing/validate-restore.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# validate-restore.sh - Script to validate Velero backup restoration
+
+set -e
+echo "Starting validation of restored resources..."
+
+# Get namespace from command line or use default
+NAMESPACE="${1:-default}"
+
+# Check all deployments
+echo "Checking deployments..."
+deployments=$(kubectl get deployment -n $NAMESPACE -o name)
+if [[ -z "$deployments" ]]; then
+  echo "❌ No deployments found in namespace $NAMESPACE"
+else
+  echo "Found deployments: $deployments"
+  
+  # Check if pods are running for each deployment
+  for deployment in $deployments; do
+    name=$(echo $deployment | cut -d'/' -f2)
+    echo -n "Checking deployment $name: "
+    
+    # Check if pods are running
+    ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
+    if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
+      echo "✅ $ready_replicas pods ready"
+    else
+      echo "❌ No pods running"
+    fi
+  done
+fi
+
+# Check all services
+echo "Checking services..."
+services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
+if [[ -z "$services" ]]; then
+  echo "❌ No services found in namespace $NAMESPACE"
+else
+  echo "Found services: $services"
+  
+  # Check if services have endpoints
+  for service in $services; do
+    name=$(echo $service | cut -d'/' -f2)
+    echo -n "Checking service $name: "
+    
+    # Check if service has endpoints
+    endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
+    if [ -n "$endpoints" ]; then
+      echo "✅ Has endpoints"
+    else
+      echo "❌ No endpoints"
+    fi
+  done
+fi
+
+# Check PVCs
+echo "Checking PVCs..."
+pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
+if [[ -z "$pvcs" ]]; then
+  echo "No PVCs found in namespace $NAMESPACE"
+else
+  pvc_count=$(echo "$pvcs" | wc -l)
+  echo "Found $pvc_count PVCs"
+  
+  # Check if PVCs are bound
+  bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
+  echo "$bound_count/$pvc_count PVCs are bound"
+  
+  if [ "$bound_count" -ne "$pvc_count" ]; then
+    echo "❌ Not all PVCs are bound"
+  else
+    echo "✅ All PVCs are bound"
+  fi
+fi
+
+echo "Validation complete!"
+exit 0