mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-09-17 10:22:54 +02:00
Ruben Hensen
405 lines
15 KiB
Bash
405 lines
15 KiB
Bash
#!/usr/bin/env bash
|
|
# beiwe-backend test for nix-infra-machine
|
|
#
|
|
# This test:
|
|
# 1. Deploys beiwe-backend with PostgreSQL, MinIO, RabbitMQ, and Celery
|
|
# 2. Verifies all services are running
|
|
# 3. Tests beiwe-backend endpoints and basic functionality
|
|
# 4. Cleans up on teardown
|
|
|
|
# Handle teardown command
|
|
if [ "$CMD" = "teardown" ]; then
|
|
echo "Tearing down beiwe-backend test..."
|
|
|
|
# Stop services
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'systemctl stop beiwe-celery-beat 2>/dev/null || true'
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'systemctl stop beiwe-celery-worker 2>/dev/null || true'
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'systemctl stop beiwe-backend 2>/dev/null || true'
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'systemctl stop minio 2>/dev/null || true'
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'systemctl stop postgresql 2>/dev/null || true'
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'systemctl stop rabbitmq 2>/dev/null || true'
|
|
|
|
# Clean up data directories
|
|
echo " Removing beiwe-backend data directory..."
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'rm -rf /var/lib/beiwe-backend'
|
|
|
|
echo " Removing MinIO data directory..."
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'rm -rf /var/lib/minio'
|
|
|
|
echo " Removing PostgreSQL data directory..."
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'rm -rf /var/lib/postgresql'
|
|
|
|
echo " Removing RabbitMQ data directory..."
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" \
|
|
'rm -rf /var/lib/rabbitmq'
|
|
|
|
echo "beiwe-backend teardown complete"
|
|
return 0
|
|
fi
|
|
|
|
|
|
# ============================================================================
|
|
# Test Setup
|
|
# ============================================================================
|
|
|
|
_start=$(date +%s)
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "Beiwe Backend Test"
|
|
echo "========================================"
|
|
echo ""
|
|
echo "This test verifies:"
|
|
echo " - PostgreSQL database connectivity"
|
|
echo " - MinIO S3-compatible storage"
|
|
echo " - RabbitMQ message broker"
|
|
echo " - Celery background task worker"
|
|
echo " - Beiwe backend web service"
|
|
echo ""
|
|
|
|
# Deploy the beiwe-backend configuration to test nodes
|
|
echo "Step 1: Deploying beiwe-backend configuration..."
|
|
$NIX_INFRA fleet deploy-apps -d "$WORK_DIR" --batch --debug --env="$ENV" \
|
|
--test-dir="$WORK_DIR/$TEST_DIR" --no-rebuild \
|
|
--target="$TARGET"
|
|
|
|
# Apply the configuration
|
|
echo "Step 2: Applying NixOS configuration..."
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" "nixos-rebuild switch --fast"
|
|
|
|
_setup=$(date +%s)
|
|
|
|
# ============================================================================
|
|
# Test Verification
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 3: Verifying deployment..."
|
|
echo ""
|
|
|
|
# Wait for services to be ready
|
|
for node in $TARGET; do
|
|
echo "Waiting for services on $node..."
|
|
|
|
# Wait for PostgreSQL
|
|
wait_for_service "$node" "postgresql" --timeout=60
|
|
wait_for_port "$node" "5432" --timeout=30
|
|
|
|
# Wait for MinIO
|
|
wait_for_service "$node" "minio" --timeout=60
|
|
wait_for_port "$node" "9000" --timeout=30
|
|
|
|
# Wait for MinIO bucket creation to complete
|
|
wait_for_service "$node" "minio-create-bucket" --timeout=60
|
|
|
|
# Wait for RabbitMQ
|
|
wait_for_service "$node" "rabbitmq" --timeout=60
|
|
wait_for_port "$node" "5672" --timeout=30
|
|
|
|
# Wait for beiwe-backend (may take time for migrations)
|
|
wait_for_service "$node" "beiwe-backend" --timeout=60
|
|
wait_for_port "$node" "8080" --timeout=30
|
|
|
|
# Wait for Celery worker (may take time to connect to RabbitMQ)
|
|
wait_for_service "$node" "beiwe-celery-worker" --timeout=90
|
|
|
|
# Wait for HTTP response (Django may take time to initialize)
|
|
wait_for_http "$node" "http://localhost:8080/" "200 302 303 400 403 404 500" --timeout=90
|
|
done
|
|
|
|
|
|
# ============================================================================
|
|
# Check Service Status
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 4: Checking systemd services status..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "...checking services on $node"
|
|
|
|
# PostgreSQL
|
|
echo " Checking PostgreSQL..."
|
|
assert_service_active "$node" "postgresql" || show_service_logs "$node" "postgresql" 50
|
|
|
|
# MinIO
|
|
echo " Checking MinIO..."
|
|
assert_service_active "$node" "minio" || show_service_logs "$node" "minio" 50
|
|
|
|
# MinIO bucket creation (oneshot service)
|
|
echo " Checking minio-create-bucket..."
|
|
bucket_setup_status=$(cmd_clean "$node" "systemctl is-active minio-create-bucket 2>/dev/null || echo 'unknown'")
|
|
if [[ "$bucket_setup_status" == "active" ]] || [[ "$bucket_setup_status" == "activating" ]]; then
|
|
echo -e " ${GREEN}✓${NC} minio-create-bucket: $bucket_setup_status [pass]"
|
|
else
|
|
echo -e " ${YELLOW}!${NC} minio-create-bucket: $bucket_setup_status [warn]"
|
|
show_service_logs "$node" "minio-create-bucket" 50
|
|
fi
|
|
|
|
# RabbitMQ
|
|
echo " Checking RabbitMQ..."
|
|
assert_service_active "$node" "rabbitmq" || show_service_logs "$node" "rabbitmq" 50
|
|
|
|
# Beiwe Backend
|
|
echo " Checking beiwe-backend..."
|
|
assert_service_active "$node" "beiwe-backend" || show_service_logs "$node" "beiwe-backend" 100
|
|
|
|
# Celery Worker
|
|
echo " Checking beiwe-celery-worker..."
|
|
assert_service_active "$node" "beiwe-celery-worker" || show_service_logs "$node" "beiwe-celery-worker" 100
|
|
|
|
# Celery Beat (scheduler)
|
|
echo " Checking beiwe-celery-beat..."
|
|
assert_service_active "$node" "beiwe-celery-beat" || show_service_logs "$node" "beiwe-celery-beat" 50
|
|
done
|
|
|
|
|
|
# ============================================================================
|
|
# Check Port Bindings
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 5: Checking port bindings..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Checking ports on $node..."
|
|
assert_port_listening "$node" "5432" "PostgreSQL port 5432"
|
|
assert_port_listening "$node" "9000" "MinIO API port 9000"
|
|
assert_port_listening "$node" "5672" "RabbitMQ AMQP port 5672"
|
|
assert_port_listening "$node" "15672" "RabbitMQ Management port 15672"
|
|
assert_port_listening "$node" "8080" "Beiwe backend port 8080"
|
|
done
|
|
|
|
# ============================================================================
|
|
# Database Tests
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 6: Testing PostgreSQL database..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Testing database on $node..."
|
|
|
|
# Check database exists
|
|
db_exists=$(cmd_clean "$node" "sudo -u postgres psql -lqt | grep -c beiwe || echo 0")
|
|
if [[ "$db_exists" -ge 1 ]]; then
|
|
echo -e " ${GREEN}✓${NC} Database 'beiwe' exists [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} Database 'beiwe' not found [fail]"
|
|
fi
|
|
|
|
# Check user exists
|
|
user_exists=$(cmd_clean "$node" "sudo -u postgres psql -c \"SELECT 1 FROM pg_roles WHERE rolname='beiwe'\" | grep -c 1 || echo 0")
|
|
if [[ "$user_exists" -ge 1 ]]; then
|
|
echo -e " ${GREEN}✓${NC} User 'beiwe' exists [pass]"
|
|
else
|
|
echo -e " ${YELLOW}!${NC} User 'beiwe' not found (may be created on first run) [warn]"
|
|
fi
|
|
done
|
|
|
|
# ============================================================================
|
|
# MinIO Tests
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 7: Testing MinIO S3 storage..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Testing MinIO on $node..."
|
|
|
|
# Check MinIO health
|
|
minio_health=$(cmd_clean "$node" "curl -s http://127.0.0.1:9000/minio/health/live 2>/dev/null || echo 'failed'")
|
|
if [[ "$minio_health" != "failed" ]]; then
|
|
echo -e " ${GREEN}✓${NC} MinIO health check passed [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} MinIO health check failed [fail]"
|
|
fi
|
|
|
|
# Check bucket exists (set HOME for mc config)
|
|
bucket_exists=$(cmd_clean "$node" "export HOME=/tmp/mc-test-check && mkdir -p \$HOME && mc alias set local http://127.0.0.1:9000 minioadmin minioadmin123 --api S3v4 > /dev/null 2>&1 && mc ls local/beiwe-data > /dev/null 2>&1 && echo 'yes' || echo 'no'")
|
|
if [[ "$bucket_exists" == "yes" ]]; then
|
|
echo -e " ${GREEN}✓${NC} Bucket 'beiwe-data' exists [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} Bucket 'beiwe-data' not found [fail]"
|
|
# Show bucket list for debugging
|
|
echo " Available buckets:"
|
|
cmd_clean "$node" "export HOME=/tmp/mc-test-check && mc ls local/ 2>/dev/null || echo ' (none)'" | while read line; do echo " $line"; done
|
|
fi
|
|
done
|
|
|
|
|
|
# ============================================================================
|
|
# RabbitMQ Tests
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 8: Testing RabbitMQ message broker..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Testing RabbitMQ on $node..."
|
|
|
|
# Check RabbitMQ via management API (more reliable than rabbitmqctl which needs root)
|
|
rabbitmq_api_status=$(cmd_clean "$node" "curl -s -o /dev/null -w '%{http_code}' -u guest:guest http://127.0.0.1:15672/api/overview 2>/dev/null || echo '000'")
|
|
if [[ "$rabbitmq_api_status" == "200" ]]; then
|
|
echo -e " ${GREEN}✓${NC} RabbitMQ API responding (status: $rabbitmq_api_status) [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} RabbitMQ API not responding (status: $rabbitmq_api_status) [fail]"
|
|
fi
|
|
|
|
# Check management UI is accessible
|
|
mgmt_status=$(cmd_clean "$node" "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:15672/ 2>/dev/null || echo '000'")
|
|
if [[ "$mgmt_status" == "200" ]] || [[ "$mgmt_status" == "301" ]]; then
|
|
echo -e " ${GREEN}✓${NC} RabbitMQ Management UI accessible (status: $mgmt_status) [pass]"
|
|
else
|
|
echo -e " ${YELLOW}!${NC} RabbitMQ Management UI returned status: $mgmt_status [warn]"
|
|
fi
|
|
done
|
|
|
|
|
|
# ============================================================================
|
|
# Celery Worker Tests
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 9: Testing Celery worker..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Testing Celery on $node..."
|
|
|
|
# Check Celery process is running
|
|
celery_running=$(cmd_clean "$node" "pgrep -f 'celery.*worker' > /dev/null && echo 'yes' || echo 'no'")
|
|
if [[ "$celery_running" == "yes" ]]; then
|
|
echo -e " ${GREEN}✓${NC} Celery worker process running [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} Celery worker process not found [fail]"
|
|
fi
|
|
|
|
# Check Celery beat (scheduler) is running
|
|
celery_beat=$(cmd_clean "$node" "pgrep -f 'celery.*beat' > /dev/null && echo 'yes' || echo 'no'")
|
|
if [[ "$celery_beat" == "yes" ]]; then
|
|
echo -e " ${GREEN}✓${NC} Celery beat (scheduler) running [pass]"
|
|
else
|
|
echo -e " ${YELLOW}!${NC} Celery beat not running (scheduled tasks may not work) [warn]"
|
|
fi
|
|
done
|
|
|
|
# ============================================================================
|
|
# Beiwe Backend HTTP Tests
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 10: Testing Beiwe backend HTTP endpoints..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Testing Beiwe backend on $node..."
|
|
|
|
# Test basic HTTP response (any response means server is running)
|
|
http_status=$(cmd_clean "$node" "curl -s -o /dev/null -w '%{http_code}' http://localhost:8080/ 2>/dev/null")
|
|
if [[ -n "$http_status" ]] && [[ "$http_status" != "000" ]]; then
|
|
echo -e " ${GREEN}✓${NC} HTTP response received (status: $http_status) [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} No HTTP response from beiwe-backend [fail]"
|
|
fi
|
|
|
|
# Test if Django is responding (check for specific patterns in response)
|
|
response_body=$(cmd_clean "$node" "curl -s http://localhost:8080/ 2>/dev/null | head -c 500")
|
|
if [[ "$response_body" == *"html"* ]] || [[ "$response_body" == *"HTML"* ]] || [[ "$response_body" == *"django"* ]] || [[ "$response_body" == *"Django"* ]] || [[ "$response_body" == *"Beiwe"* ]] || [[ "$response_body" == *"beiwe"* ]]; then
|
|
echo -e " ${GREEN}✓${NC} Django/Beiwe response detected [pass]"
|
|
else
|
|
echo -e " ${YELLOW}!${NC} Response doesn't look like Django/Beiwe (may still be OK) [warn]"
|
|
echo " Response preview: ${response_body:0:100}..."
|
|
fi
|
|
|
|
# Check that gunicorn process is running
|
|
echo " Checking gunicorn process..."
|
|
gunicorn_running=$(cmd_clean "$node" "pgrep -f gunicorn > /dev/null && echo 'yes' || echo 'no'")
|
|
if [[ "$gunicorn_running" == "yes" ]]; then
|
|
echo -e " ${GREEN}✓${NC} Gunicorn process running [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} Gunicorn process not found [fail]"
|
|
fi
|
|
done
|
|
|
|
# ============================================================================
|
|
# Service Health Summary
|
|
# ============================================================================
|
|
|
|
echo ""
|
|
echo "Step 11: Final health checks..."
|
|
echo ""
|
|
|
|
for node in $TARGET; do
|
|
echo "Final checks on $node..."
|
|
|
|
# Check for any failed units
|
|
echo " Checking for failed units..."
|
|
failed_units=$(cmd_clean "$node" "systemctl list-units --failed | grep -E 'beiwe|minio|postgresql|rabbitmq' || echo 'none'")
|
|
if [[ "$failed_units" == *"none"* ]] || [[ -z "$failed_units" ]] || [[ ! "$failed_units" == *"failed"* ]]; then
|
|
echo -e " ${GREEN}✓${NC} No failed related units [pass]"
|
|
else
|
|
echo -e " ${RED}✗${NC} Failed units found: $failed_units [fail]"
|
|
fi
|
|
|
|
# Check data directories
|
|
echo " Checking data directories..."
|
|
assert_dir_exists "$node" "/var/lib/minio" "MinIO data directory"
|
|
done
|
|
|
|
# ============================================================================
|
|
# Test Summary
|
|
# ============================================================================
|
|
|
|
_end=$(date +%s)
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "Beiwe Backend Test Summary"
|
|
echo "========================================"
|
|
echo ""
|
|
echo "Services tested:"
|
|
echo " ✓ PostgreSQL (database)"
|
|
echo " ✓ MinIO (S3-compatible storage)"
|
|
echo " ✓ RabbitMQ (message broker)"
|
|
echo " ✓ Celery Worker (background tasks)"
|
|
echo " ✓ Celery Beat (task scheduler)"
|
|
echo " ✓ Beiwe Backend (Django/Gunicorn)"
|
|
echo ""
|
|
echo "Optional services NOT configured:"
|
|
echo " ✗ Firebase credentials (for push notifications)"
|
|
echo " ✗ Sentry error tracking"
|
|
echo ""
|
|
echo "Features available with this configuration:"
|
|
echo " ✓ Web-based study management portal"
|
|
echo " ✓ User authentication and management"
|
|
echo " ✓ Study and survey configuration"
|
|
echo " ✓ Participant registration"
|
|
echo " ✓ Data uploads from mobile apps"
|
|
echo " ✓ Background task processing"
|
|
echo " ✓ Data processing pipelines"
|
|
echo ""
|
|
|
|
printf '+ setup %s\n' $(printTime $_start $_setup)
|
|
printf '+ tests %s\n' $(printTime $_setup $_end)
|
|
printf '= TOTAL %s\n' $(printTime $_start $_end)
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "Beiwe Backend Test Complete"
|
|
echo "========================================"
|