mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-09-17 02:12:55 +02:00
Add backup testing scripts
This commit is contained in:
Ruben Hensen
@@ -39,6 +39,15 @@ This script will look for credentials in `~/.velero/credentials-velero` or promp
|
||||
|
||||
Execute the test script with appropriate parameters:
|
||||
|
||||
```bash
|
||||
./test-velero-backup.sh \
|
||||
--s3-access-key <ACCESS_KEY> \
|
||||
--s3-secret-key <SECRET_KEY> \
|
||||
--backup-name <BACKUP_NAME> \
|
||||
--original-namespace mbgwp \
|
||||
--test-namespace mbgwp
|
||||
```
|
||||
|
||||
```bash
|
||||
./test-velero-backup.sh \
|
||||
--s3-access-key YOUR_ACCESS_KEY \
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
# any name can be used; Velero uses the labels (below)
|
||||
# to identify it rather than the name
|
||||
name: change-storage-class-config
|
||||
# must be in the velero namespace
|
||||
namespace: velero
|
||||
# the below labels should be used verbatim in your
|
||||
# ConfigMap.
|
||||
labels:
|
||||
# this value-less label identifies the ConfigMap as
|
||||
# config for a plugin (i.e. the built-in change storage
|
||||
# class restore item action plugin)
|
||||
velero.io/plugin-config: ""
|
||||
# this label identifies the name and kind of plugin
|
||||
# that this ConfigMap is for.
|
||||
velero.io/change-storage-class: RestoreItemAction
|
||||
data:
|
||||
# add 1+ key-value pairs here, where the key is the old
|
||||
# storage class name and the value is the new storage
|
||||
# class name.
|
||||
longhorn: local-path
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/bin/bash
|
||||
# setup-credentials.sh - Script to help setup credentials for Velero testing
|
||||
|
||||
# Directory where Velero credentials are stored on the production system
|
||||
# This would be your production system's credentials for accessing MinIO
|
||||
VELERO_CREDS_PATH="${HOME}/.velero/credentials-velero"
|
||||
|
||||
if [ -f "${VELERO_CREDS_PATH}" ]; then
|
||||
echo "Found Velero credentials at ${VELERO_CREDS_PATH}"
|
||||
# Extract AWS credentials from the file
|
||||
AWS_ACCESS_KEY_ID=$(grep aws_access_key_id ${VELERO_CREDS_PATH} | cut -d= -f2 | tr -d '[:space:]')
|
||||
AWS_SECRET_ACCESS_KEY=$(grep aws_secret_access_key ${VELERO_CREDS_PATH} | cut -d= -f2 | tr -d '[:space:]')
|
||||
|
||||
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then
|
||||
echo "Credentials found and will be used for backup testing"
|
||||
|
||||
# Store them temporarily for the test script to use
|
||||
cat > /tmp/velero-test-credentials <<EOF
|
||||
[default]
|
||||
aws_access_key_id=${AWS_ACCESS_KEY_ID}
|
||||
aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
|
||||
EOF
|
||||
|
||||
echo "Credentials saved to /tmp/velero-test-credentials"
|
||||
echo ""
|
||||
echo "Run your test script with:"
|
||||
echo "./test-velero-backup.sh --s3-access-key ${AWS_ACCESS_KEY_ID} --s3-secret-key ${AWS_SECRET_ACCESS_KEY} [other options]"
|
||||
else
|
||||
echo "Error: Could not extract credentials from ${VELERO_CREDS_PATH}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Error: Velero credentials file not found at ${VELERO_CREDS_PATH}"
|
||||
echo "Please create this file with the following format:"
|
||||
echo "[default]"
|
||||
echo "aws_access_key_id=YOUR_MINIO_ACCESS_KEY"
|
||||
echo "aws_secret_access_key=YOUR_MINIO_SECRET_KEY"
|
||||
exit 1
|
||||
fi
|
||||
@@ -76,7 +76,7 @@ while [[ $# -gt 0 ]]; do
|
||||
done
|
||||
|
||||
# Verify required variables
|
||||
if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" ]]; then
|
||||
if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" || -z "$ORIGINAL_NAMESPACE" || -z "$TEST_NAMESPACE" ]]; then
|
||||
echo "Error: S3 configuration is incomplete"
|
||||
exit 1
|
||||
fi
|
||||
@@ -106,29 +106,34 @@ k3d cluster create backup-test \
|
||||
--api-port 6443 \
|
||||
--port "80:80@loadbalancer" \
|
||||
--port "443:443@loadbalancer" \
|
||||
--agents 2 \
|
||||
--agents 0 \
|
||||
--k3s-arg "--disable=traefik@server:0" \
|
||||
--wait
|
||||
# -v /mnt/longhorn:/var/lib/longhorn:shared@all \
|
||||
# -i hebury/k3s:v1.32.2-k3s1 \
|
||||
# --wait
|
||||
|
||||
# Verify cluster is running
|
||||
kubectl config use-context k3d-backup-test
|
||||
kubectl get nodes
|
||||
|
||||
# 2. Install Longhorn
|
||||
echo "Installing Longhorn..."
|
||||
kubectl create namespace longhorn-system
|
||||
# echo "Installing Longhorn..."
|
||||
# kubectl create namespace longhorn-system
|
||||
|
||||
helm repo add longhorn https://charts.longhorn.io
|
||||
helm repo update
|
||||
# helm repo add longhorn https://charts.longhorn.io
|
||||
# helm repo update
|
||||
|
||||
helm install longhorn longhorn/longhorn \
|
||||
--namespace longhorn-system \
|
||||
--set persistence.defaultClassReplicaCount=1 \
|
||||
--set defaultSettings.backupTarget="" \
|
||||
--set defaultSettings.defaultReplicaCount=1
|
||||
# helm install longhorn longhorn/longhorn \
|
||||
# --namespace longhorn-system \
|
||||
# --set persistence.defaultClassReplicaCount=1 \
|
||||
# --set defaultSettings.backupTarget="" \
|
||||
# --set defaultSettings.defaultReplicaCount=1
|
||||
|
||||
echo "Waiting for Longhorn to be ready..."
|
||||
kubectl -n longhorn-system rollout status deployment/longhorn-ui
|
||||
# echo "Waiting for Longhorn to be ready..."
|
||||
# kubectl -n longhorn-system rollout status deployment/longhorn-ui
|
||||
# kubectl -n longhorn-system rollout status deployment/longhorn-driver-deployer
|
||||
# kubectl -n longhorn-system rollout status daemonset/longhorn-manager
|
||||
|
||||
# 3. Install Velero with S3 provider
|
||||
echo "Installing Velero..."
|
||||
@@ -174,6 +179,9 @@ rm velero-credentials
|
||||
echo "Waiting for Velero to be fully ready..."
|
||||
kubectl -n velero rollout status deployment/velero --timeout=120s
|
||||
|
||||
echo "Install change-storage-class.yaml"
|
||||
kubectl apply -f change-storage-class.yaml
|
||||
|
||||
# Check if velero CLI is installed and functioning
|
||||
if ! command -v velero &> /dev/null; then
|
||||
echo "Error: Velero CLI is not installed or not in PATH"
|
||||
@@ -262,6 +270,8 @@ else
|
||||
velero restore create --from-backup $BACKUP_NAME \
|
||||
--namespace-mappings $ORIGINAL_NAMESPACE:$TEST_NAMESPACE \
|
||||
--include-namespaces $ORIGINAL_NAMESPACE \
|
||||
--exclude-namespaces kube-system,kube-public,kube-node-lease,velero \
|
||||
--exclude-resources certificates.cert-manager.io \
|
||||
--wait
|
||||
fi
|
||||
|
||||
@@ -274,89 +284,89 @@ echo "Waiting for pods to be ready..."
|
||||
kubectl wait --for=condition=ready pod --all -n $TEST_NAMESPACE --timeout=300s || true
|
||||
|
||||
# 8. Run validation script (create it first if it doesn't exist)
|
||||
if [[ ! -f ./validate-restore.sh ]]; then
|
||||
echo "Creating validation script..."
|
||||
cat > validate-restore.sh <<'EOF'
|
||||
#!/bin/bash
|
||||
# validate-restore.sh - Script to validate Velero backup restoration
|
||||
# if [[ ! -f ./validate-restore.sh ]]; then
|
||||
# echo "Creating validation script..."
|
||||
# cat > validate-restore.sh <<'EOF'
|
||||
# #!/bin/bash
|
||||
# # validate-restore.sh - Script to validate Velero backup restoration
|
||||
|
||||
set -e
|
||||
echo "Starting validation of restored resources..."
|
||||
# set -e
|
||||
# echo "Starting validation of restored resources..."
|
||||
|
||||
# Get namespace from command line or use default
|
||||
NAMESPACE="${1:-default}"
|
||||
# # Get namespace from command line or use default
|
||||
# NAMESPACE="${1:-default}"
|
||||
|
||||
# Check all deployments
|
||||
echo "Checking deployments..."
|
||||
deployments=$(kubectl get deployment -n $NAMESPACE -o name)
|
||||
if [[ -z "$deployments" ]]; then
|
||||
echo "❌ No deployments found in namespace $NAMESPACE"
|
||||
else
|
||||
echo "Found deployments: $deployments"
|
||||
# # Check all deployments
|
||||
# echo "Checking deployments..."
|
||||
# deployments=$(kubectl get deployment -n $NAMESPACE -o name)
|
||||
# if [[ -z "$deployments" ]]; then
|
||||
# echo "❌ No deployments found in namespace $NAMESPACE"
|
||||
# else
|
||||
# echo "Found deployments: $deployments"
|
||||
|
||||
# Check if pods are running for each deployment
|
||||
for deployment in $deployments; do
|
||||
name=$(echo $deployment | cut -d'/' -f2)
|
||||
echo -n "Checking deployment $name: "
|
||||
# # Check if pods are running for each deployment
|
||||
# for deployment in $deployments; do
|
||||
# name=$(echo $deployment | cut -d'/' -f2)
|
||||
# echo -n "Checking deployment $name: "
|
||||
|
||||
# Check if pods are running
|
||||
ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
|
||||
if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
|
||||
echo "✅ $ready_replicas pods ready"
|
||||
else
|
||||
echo "❌ No pods running"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
# # Check if pods are running
|
||||
# ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
|
||||
# if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
|
||||
# echo "✅ $ready_replicas pods ready"
|
||||
# else
|
||||
# echo "❌ No pods running"
|
||||
# fi
|
||||
# done
|
||||
# fi
|
||||
|
||||
# Check all services
|
||||
echo "Checking services..."
|
||||
services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
|
||||
if [[ -z "$services" ]]; then
|
||||
echo "❌ No services found in namespace $NAMESPACE"
|
||||
else
|
||||
echo "Found services: $services"
|
||||
# # Check all services
|
||||
# echo "Checking services..."
|
||||
# services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
|
||||
# if [[ -z "$services" ]]; then
|
||||
# echo "❌ No services found in namespace $NAMESPACE"
|
||||
# else
|
||||
# echo "Found services: $services"
|
||||
|
||||
# Check if services have endpoints
|
||||
for service in $services; do
|
||||
name=$(echo $service | cut -d'/' -f2)
|
||||
echo -n "Checking service $name: "
|
||||
# # Check if services have endpoints
|
||||
# for service in $services; do
|
||||
# name=$(echo $service | cut -d'/' -f2)
|
||||
# echo -n "Checking service $name: "
|
||||
|
||||
# Check if service has endpoints
|
||||
endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
|
||||
if [ -n "$endpoints" ]; then
|
||||
echo "✅ Has endpoints"
|
||||
else
|
||||
echo "❌ No endpoints"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
# # Check if service has endpoints
|
||||
# endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
|
||||
# if [ -n "$endpoints" ]; then
|
||||
# echo "✅ Has endpoints"
|
||||
# else
|
||||
# echo "❌ No endpoints"
|
||||
# fi
|
||||
# done
|
||||
# fi
|
||||
|
||||
# Check PVCs
|
||||
echo "Checking PVCs..."
|
||||
pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
|
||||
if [[ -z "$pvcs" ]]; then
|
||||
echo "No PVCs found in namespace $NAMESPACE"
|
||||
else
|
||||
pvc_count=$(echo "$pvcs" | wc -l)
|
||||
echo "Found $pvc_count PVCs"
|
||||
# # Check PVCs
|
||||
# echo "Checking PVCs..."
|
||||
# pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
|
||||
# if [[ -z "$pvcs" ]]; then
|
||||
# echo "No PVCs found in namespace $NAMESPACE"
|
||||
# else
|
||||
# pvc_count=$(echo "$pvcs" | wc -l)
|
||||
# echo "Found $pvc_count PVCs"
|
||||
|
||||
# Check if PVCs are bound
|
||||
bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
|
||||
echo "$bound_count/$pvc_count PVCs are bound"
|
||||
# # Check if PVCs are bound
|
||||
# bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
|
||||
# echo "$bound_count/$pvc_count PVCs are bound"
|
||||
|
||||
if [ "$bound_count" -ne "$pvc_count" ]; then
|
||||
echo "❌ Not all PVCs are bound"
|
||||
else
|
||||
echo "✅ All PVCs are bound"
|
||||
fi
|
||||
fi
|
||||
# if [ "$bound_count" -ne "$pvc_count" ]; then
|
||||
# echo "❌ Not all PVCs are bound"
|
||||
# else
|
||||
# echo "✅ All PVCs are bound"
|
||||
# fi
|
||||
# fi
|
||||
|
||||
echo "Validation complete!"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x validate-restore.sh
|
||||
fi
|
||||
# echo "Validation complete!"
|
||||
# exit 0
|
||||
# EOF
|
||||
# chmod +x validate-restore.sh
|
||||
# fi
|
||||
|
||||
echo "Running validation tests..."
|
||||
./validate-restore.sh $TEST_NAMESPACE
|
||||
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# validate-restore.sh - Script to validate Velero backup restoration
|
||||
|
||||
set -e
|
||||
echo "Starting validation of restored resources..."
|
||||
|
||||
# Get namespace from command line or use default
|
||||
NAMESPACE="${1:-default}"
|
||||
|
||||
# Check all deployments
|
||||
echo "Checking deployments..."
|
||||
deployments=$(kubectl get deployment -n $NAMESPACE -o name)
|
||||
if [[ -z "$deployments" ]]; then
|
||||
echo "❌ No deployments found in namespace $NAMESPACE"
|
||||
else
|
||||
echo "Found deployments: $deployments"
|
||||
|
||||
# Check if pods are running for each deployment
|
||||
for deployment in $deployments; do
|
||||
name=$(echo $deployment | cut -d'/' -f2)
|
||||
echo -n "Checking deployment $name: "
|
||||
|
||||
# Check if pods are running
|
||||
ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
|
||||
if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
|
||||
echo "✅ $ready_replicas pods ready"
|
||||
else
|
||||
echo "❌ No pods running"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Check all services
|
||||
echo "Checking services..."
|
||||
services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
|
||||
if [[ -z "$services" ]]; then
|
||||
echo "❌ No services found in namespace $NAMESPACE"
|
||||
else
|
||||
echo "Found services: $services"
|
||||
|
||||
# Check if services have endpoints
|
||||
for service in $services; do
|
||||
name=$(echo $service | cut -d'/' -f2)
|
||||
echo -n "Checking service $name: "
|
||||
|
||||
# Check if service has endpoints
|
||||
endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
|
||||
if [ -n "$endpoints" ]; then
|
||||
echo "✅ Has endpoints"
|
||||
else
|
||||
echo "❌ No endpoints"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Check PVCs
|
||||
echo "Checking PVCs..."
|
||||
pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
|
||||
if [[ -z "$pvcs" ]]; then
|
||||
echo "No PVCs found in namespace $NAMESPACE"
|
||||
else
|
||||
pvc_count=$(echo "$pvcs" | wc -l)
|
||||
echo "Found $pvc_count PVCs"
|
||||
|
||||
# Check if PVCs are bound
|
||||
bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
|
||||
echo "$bound_count/$pvc_count PVCs are bound"
|
||||
|
||||
if [ "$bound_count" -ne "$pvc_count" ]; then
|
||||
echo "❌ Not all PVCs are bound"
|
||||
else
|
||||
echo "✅ All PVCs are bound"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Validation complete!"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user