mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-09-17 10:22:54 +02:00
Ruben Hensen
346 lines
8.6 KiB
Bash
346 lines
8.6 KiB
Bash
#!/usr/bin/env bash
|
|
# Shared helper functions for nix-infra-machine tests
|
|
|
|
# ============================================================================
|
|
# Colors for Test Output
|
|
# ============================================================================
|
|
|
|
GREEN='\033[0;32m'
|
|
RED='\033[0;31m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# ============================================================================
|
|
# Utility Functions
|
|
# ============================================================================
|
|
|
|
appendWithLineBreak() {
|
|
if [ -z "$1" ]; then
|
|
printf '%s' "$2"
|
|
else
|
|
printf '%s\n%s' "$1" "$2"
|
|
fi
|
|
}
|
|
|
|
cmd() {
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$1" "$2"
|
|
}
|
|
|
|
# Get command output with node prefix stripped and whitespace trimmed
|
|
# Use for single values that need arithmetic or exact comparison
|
|
# Example: count=$(cmd_value "$node" "pgrep -c redis-server || echo 0")
|
|
cmd_value() {
|
|
local node="$1"
|
|
local command="$2"
|
|
local output
|
|
output=$(cmd "$node" "$command")
|
|
# Strip "nodename: " prefix and trim whitespace
|
|
echo "$output" | sed "s/^${node}: //" | tr -d '[:space:]'
|
|
}
|
|
|
|
# Get command output with node prefix stripped but preserving structure
|
|
# Use for multi-line output or when whitespace matters
|
|
# Example: config=$(cmd_clean "$node" "cat /etc/config")
|
|
cmd_clean() {
|
|
local node="$1"
|
|
local command="$2"
|
|
local output
|
|
output=$(cmd "$node" "$command")
|
|
# Strip "nodename: " prefix from each line
|
|
echo "$output" | sed "s/^${node}: //"
|
|
}
|
|
|
|
|
|
printTime() {
|
|
local _start=$1; local _end=$2; local _secs=$((_end-_start))
|
|
printf '%02dh:%02dm:%02ds' $((_secs/3600)) $((_secs%3600/60)) $((_secs%60))
|
|
}
|
|
|
|
# ============================================================================
|
|
# Common Commands (used by run-tests.sh command parsing)
|
|
# ============================================================================
|
|
|
|
if [ "$CMD" = "pull" ]; then
|
|
git -C "$WORK_DIR" pull
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$CMD" = "ssh" ]; then
|
|
if [ -z "$REST" ]; then
|
|
echo "Usage: $0 ssh --env=$ENV [node]"
|
|
exit 1
|
|
fi
|
|
HCLOUD_TOKEN=$HCLOUD_TOKEN hcloud server ssh $REST -i "$WORK_DIR/ssh/$SSH_KEY"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$CMD" = "cmd" ]; then
|
|
if [ -z "$TARGET" ] || [ -z "$REST" ]; then
|
|
echo "Usage: $0 cmd --env=$ENV --target=[node] [cmd goes here]"
|
|
exit 1
|
|
fi
|
|
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" "$REST"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$CMD" = "action" ]; then
|
|
if [ -z "$TARGET" ] || [ -z "$REST" ]; then
|
|
echo "Usage: $0 action --env=$ENV --target=[node] [module] [cmd]"
|
|
exit 1
|
|
fi
|
|
|
|
read -r module action_cmd <<< "$REST"
|
|
$NIX_INFRA fleet action -d "$WORK_DIR" --target="$TARGET" --app-module="$module" \
|
|
--cmd="$action_cmd"
|
|
exit 0
|
|
fi
|
|
|
|
# ============================================================================
|
|
# Health Check Functions
|
|
# ============================================================================
|
|
|
|
checkNixos() {
|
|
echo "Checking NixOS..."
|
|
local NODES="$1"
|
|
local node
|
|
local _nixos_fail=""
|
|
|
|
for node in $NODES; do
|
|
local output=$(cmd "$node" "uname -a" 2>&1)
|
|
local result="$output"
|
|
if [[ "$result" == *"NixOS"* ]]; then
|
|
echo " ✓ nixos: ok ($node)"
|
|
else
|
|
echo " ✗ nixos: fail ($node)"
|
|
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
|
echo " $output"
|
|
fi
|
|
_nixos_fail="true"
|
|
fi
|
|
done
|
|
|
|
if [ -n "$_nixos_fail" ]; then
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
checkPodman() {
|
|
echo "Checking Podman..."
|
|
local NODES="$1"
|
|
local node
|
|
local _failed=""
|
|
|
|
for node in $NODES; do
|
|
local output=$(cmd "$node" "podman --version" 2>&1)
|
|
local result="$output"
|
|
if [[ "$result" == *"podman version"* ]]; then
|
|
echo " ✓ podman: ok ($node)"
|
|
else
|
|
echo " ✗ podman: not installed or not running ($node)"
|
|
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
|
echo " $output"
|
|
fi
|
|
_failed="yes"
|
|
fi
|
|
done
|
|
|
|
if [ -n "$_failed" ]; then
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
checkService() {
|
|
local NODE="$1"
|
|
local SERVICE="$2"
|
|
local output=$(cmd "$NODE" "systemctl is-active $SERVICE" 2>&1)
|
|
local result="$output"
|
|
if [[ "$result" == *"active"* ]]; then
|
|
echo " ✓ $SERVICE: active ($NODE)"
|
|
return 0
|
|
else
|
|
echo " ✗ $SERVICE: inactive ($NODE)"
|
|
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
|
echo " $output"
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
checkServiceOnNodes() {
|
|
echo "Checking service: $2"
|
|
local NODES="$1"
|
|
local SERVICE="$2"
|
|
local node
|
|
local _failed=""
|
|
|
|
for node in $NODES; do
|
|
if ! checkService "$node" "$SERVICE"; then
|
|
_failed="yes"
|
|
fi
|
|
done
|
|
|
|
if [ -n "$_failed" ]; then
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
checkHttpEndpoint() {
|
|
local NODE="$1"
|
|
local URL="$2"
|
|
local EXPECTED="$3"
|
|
|
|
local output=$(cmd "$NODE" "curl -s --max-time 5 '$URL'" 2>&1)
|
|
local result="$output"
|
|
if [[ "$result" == *"$EXPECTED"* ]]; then
|
|
echo " ✓ HTTP $URL: ok ($NODE)"
|
|
return 0
|
|
else
|
|
echo " ✗ HTTP $URL: expected '$EXPECTED' ($NODE)"
|
|
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
|
echo " $output"
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
checkTcpPort() {
|
|
local NODE="$1"
|
|
local HOST="$2"
|
|
local PORT="$3"
|
|
|
|
local output=$(cmd "$NODE" "nc -zv $HOST $PORT 2>&1")
|
|
local result="$output"
|
|
if [[ "$result" == *"succeeded"* ]] || [[ "$result" == *"open"* ]] || [[ "$result" == *"Connected"* ]]; then
|
|
echo " ✓ TCP $HOST:$PORT: open ($NODE)"
|
|
return 0
|
|
else
|
|
echo " ✗ TCP $HOST:$PORT: closed ($NODE)"
|
|
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
|
echo " $output"
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# Fleet Test Function
|
|
# ============================================================================
|
|
|
|
testFleet() {
|
|
local NODES="$1"
|
|
echo "=========================================="
|
|
echo "Running Fleet Health Checks"
|
|
echo "=========================================="
|
|
|
|
local _failed=""
|
|
|
|
if ! checkNixos "$NODES"; then
|
|
_failed="yes"
|
|
fi
|
|
|
|
echo "=========================================="
|
|
if [ -n "$_failed" ]; then
|
|
echo "Health checks: FAILED"
|
|
return 1
|
|
else
|
|
echo "Health checks: PASSED"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# Container/Pod Test Helpers
|
|
# ============================================================================
|
|
|
|
waitForContainer() {
|
|
local NODE="$1"
|
|
local CONTAINER="$2"
|
|
local TIMEOUT="${3:-60}"
|
|
|
|
echo "Waiting for container $CONTAINER on $NODE (timeout: ${TIMEOUT}s)..."
|
|
local elapsed=0
|
|
local last_output=""
|
|
while [ $elapsed -lt $TIMEOUT ]; do
|
|
last_output=$(cmd "$NODE" "podman ps --filter name=$CONTAINER --format '{{.Status}}'" 2>&1)
|
|
local status="$last_output"
|
|
if [[ "$status" == *"Up"* ]]; then
|
|
echo " ✓ Container $CONTAINER is running"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
|
|
echo " ✗ Container $CONTAINER did not start within ${TIMEOUT}s"
|
|
if [ -n "$last_output" ] && [[ "$last_output" == ERROR:* ]]; then
|
|
echo " $last_output"
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
waitForService() {
|
|
local NODE="$1"
|
|
local SERVICE="$2"
|
|
local TIMEOUT="${3:-60}"
|
|
|
|
echo "Waiting for service $SERVICE on $NODE (timeout: ${TIMEOUT}s)..."
|
|
local elapsed=0
|
|
local last_output=""
|
|
while [ $elapsed -lt $TIMEOUT ]; do
|
|
last_output=$(cmd "$NODE" "systemctl is-active $SERVICE" 2>&1)
|
|
local status="$last_output"
|
|
if [[ "$status" == *"active"* ]]; then
|
|
echo " ✓ Service $SERVICE is active"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
|
|
echo " ✗ Service $SERVICE did not become active within ${TIMEOUT}s"
|
|
if [ -n "$last_output" ] && [[ "$last_output" == ERROR:* ]]; then
|
|
echo " $last_output"
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
getContainerLogs() {
|
|
local NODE="$1"
|
|
local CONTAINER="$2"
|
|
local LINES="${3:-50}"
|
|
|
|
echo "--- Container logs for $CONTAINER on $NODE ---"
|
|
cmd "$NODE" "podman logs --tail $LINES $CONTAINER" 2>&1
|
|
echo "--- End of logs ---"
|
|
}
|
|
|
|
getServiceLogs() {
|
|
local NODE="$1"
|
|
local SERVICE="$2"
|
|
local LINES="${3:-50}"
|
|
|
|
echo "--- Service logs for $SERVICE on $NODE ---"
|
|
cmd "$NODE" "journalctl -n $LINES -u $SERVICE" 2>&1
|
|
echo "--- End of logs ---"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Info output
|
|
# ============================================================================
|
|
|
|
# Print info line (not pass/fail)
|
|
# Usage: print_info "label" "value"
|
|
print_info() {
|
|
local label="$1"
|
|
local value="$2"
|
|
|
|
echo -e " ${GREEN}✓${NC} $label: $value [info]"
|
|
}
|
|
|
|
# Print cleanup success
|
|
# Usage: print_cleanup "label"
|
|
print_cleanup() {
|
|
local label="$1"
|
|
|
|
echo -e " ${GREEN}✓${NC} $label [pass]"
|
|
} |