mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-09-18 18:42:56 +02:00
Add nix-infra-machine
This commit is contained in:
Ruben Hensen
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared helper functions for nix-infra-machine tests
|
||||
|
||||
# ============================================================================
|
||||
# Colors for Test Output
|
||||
# ============================================================================
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ============================================================================
|
||||
# Utility Functions
|
||||
# ============================================================================
|
||||
|
||||
appendWithLineBreak() {
|
||||
if [ -z "$1" ]; then
|
||||
printf '%s' "$2"
|
||||
else
|
||||
printf '%s\n%s' "$1" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd() {
|
||||
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$1" "$2"
|
||||
}
|
||||
|
||||
# Get command output with node prefix stripped and whitespace trimmed
|
||||
# Use for single values that need arithmetic or exact comparison
|
||||
# Example: count=$(cmd_value "$node" "pgrep -c redis-server || echo 0")
|
||||
cmd_value() {
|
||||
local node="$1"
|
||||
local command="$2"
|
||||
local output
|
||||
output=$(cmd "$node" "$command")
|
||||
# Strip "nodename: " prefix and trim whitespace
|
||||
echo "$output" | sed "s/^${node}: //" | tr -d '[:space:]'
|
||||
}
|
||||
|
||||
# Get command output with node prefix stripped but preserving structure
|
||||
# Use for multi-line output or when whitespace matters
|
||||
# Example: config=$(cmd_clean "$node" "cat /etc/config")
|
||||
cmd_clean() {
|
||||
local node="$1"
|
||||
local command="$2"
|
||||
local output
|
||||
output=$(cmd "$node" "$command")
|
||||
# Strip "nodename: " prefix from each line
|
||||
echo "$output" | sed "s/^${node}: //"
|
||||
}
|
||||
|
||||
|
||||
printTime() {
|
||||
local _start=$1; local _end=$2; local _secs=$((_end-_start))
|
||||
printf '%02dh:%02dm:%02ds' $((_secs/3600)) $((_secs%3600/60)) $((_secs%60))
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Common Commands (used by run-tests.sh command parsing)
|
||||
# ============================================================================
|
||||
|
||||
if [ "$CMD" = "pull" ]; then
|
||||
git -C "$WORK_DIR" pull
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$CMD" = "ssh" ]; then
|
||||
if [ -z "$REST" ]; then
|
||||
echo "Usage: $0 ssh --env=$ENV [node]"
|
||||
exit 1
|
||||
fi
|
||||
HCLOUD_TOKEN=$HCLOUD_TOKEN hcloud server ssh $REST -i "$WORK_DIR/ssh/$SSH_KEY"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$CMD" = "cmd" ]; then
|
||||
if [ -z "$TARGET" ] || [ -z "$REST" ]; then
|
||||
echo "Usage: $0 cmd --env=$ENV --target=[node] [cmd goes here]"
|
||||
exit 1
|
||||
fi
|
||||
$NIX_INFRA fleet cmd -d "$WORK_DIR" --target="$TARGET" "$REST"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$CMD" = "action" ]; then
|
||||
if [ -z "$TARGET" ] || [ -z "$REST" ]; then
|
||||
echo "Usage: $0 action --env=$ENV --target=[node] [module] [cmd]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read -r module action_cmd <<< "$REST"
|
||||
$NIX_INFRA fleet action -d "$WORK_DIR" --target="$TARGET" --app-module="$module" \
|
||||
--cmd="$action_cmd"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Health Check Functions
|
||||
# ============================================================================
|
||||
|
||||
checkNixos() {
|
||||
echo "Checking NixOS..."
|
||||
local NODES="$1"
|
||||
local node
|
||||
local _nixos_fail=""
|
||||
|
||||
for node in $NODES; do
|
||||
local output=$(cmd "$node" "uname -a" 2>&1)
|
||||
local result="$output"
|
||||
if [[ "$result" == *"NixOS"* ]]; then
|
||||
echo " ✓ nixos: ok ($node)"
|
||||
else
|
||||
echo " ✗ nixos: fail ($node)"
|
||||
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
||||
echo " $output"
|
||||
fi
|
||||
_nixos_fail="true"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$_nixos_fail" ]; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkPodman() {
|
||||
echo "Checking Podman..."
|
||||
local NODES="$1"
|
||||
local node
|
||||
local _failed=""
|
||||
|
||||
for node in $NODES; do
|
||||
local output=$(cmd "$node" "podman --version" 2>&1)
|
||||
local result="$output"
|
||||
if [[ "$result" == *"podman version"* ]]; then
|
||||
echo " ✓ podman: ok ($node)"
|
||||
else
|
||||
echo " ✗ podman: not installed or not running ($node)"
|
||||
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
||||
echo " $output"
|
||||
fi
|
||||
_failed="yes"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$_failed" ]; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkService() {
|
||||
local NODE="$1"
|
||||
local SERVICE="$2"
|
||||
local output=$(cmd "$NODE" "systemctl is-active $SERVICE" 2>&1)
|
||||
local result="$output"
|
||||
if [[ "$result" == *"active"* ]]; then
|
||||
echo " ✓ $SERVICE: active ($NODE)"
|
||||
return 0
|
||||
else
|
||||
echo " ✗ $SERVICE: inactive ($NODE)"
|
||||
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
||||
echo " $output"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkServiceOnNodes() {
|
||||
echo "Checking service: $2"
|
||||
local NODES="$1"
|
||||
local SERVICE="$2"
|
||||
local node
|
||||
local _failed=""
|
||||
|
||||
for node in $NODES; do
|
||||
if ! checkService "$node" "$SERVICE"; then
|
||||
_failed="yes"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$_failed" ]; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkHttpEndpoint() {
|
||||
local NODE="$1"
|
||||
local URL="$2"
|
||||
local EXPECTED="$3"
|
||||
|
||||
local output=$(cmd "$NODE" "curl -s --max-time 5 '$URL'" 2>&1)
|
||||
local result="$output"
|
||||
if [[ "$result" == *"$EXPECTED"* ]]; then
|
||||
echo " ✓ HTTP $URL: ok ($NODE)"
|
||||
return 0
|
||||
else
|
||||
echo " ✗ HTTP $URL: expected '$EXPECTED' ($NODE)"
|
||||
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
||||
echo " $output"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkTcpPort() {
|
||||
local NODE="$1"
|
||||
local HOST="$2"
|
||||
local PORT="$3"
|
||||
|
||||
local output=$(cmd "$NODE" "nc -zv $HOST $PORT 2>&1")
|
||||
local result="$output"
|
||||
if [[ "$result" == *"succeeded"* ]] || [[ "$result" == *"open"* ]] || [[ "$result" == *"Connected"* ]]; then
|
||||
echo " ✓ TCP $HOST:$PORT: open ($NODE)"
|
||||
return 0
|
||||
else
|
||||
echo " ✗ TCP $HOST:$PORT: closed ($NODE)"
|
||||
if [ -n "$output" ] && [[ "$output" == ERROR:* ]]; then
|
||||
echo " $output"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Fleet Test Function
|
||||
# ============================================================================
|
||||
|
||||
testFleet() {
|
||||
local NODES="$1"
|
||||
echo "=========================================="
|
||||
echo "Running Fleet Health Checks"
|
||||
echo "=========================================="
|
||||
|
||||
local _failed=""
|
||||
|
||||
if ! checkNixos "$NODES"; then
|
||||
_failed="yes"
|
||||
fi
|
||||
|
||||
echo "=========================================="
|
||||
if [ -n "$_failed" ]; then
|
||||
echo "Health checks: FAILED"
|
||||
return 1
|
||||
else
|
||||
echo "Health checks: PASSED"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Container/Pod Test Helpers
|
||||
# ============================================================================
|
||||
|
||||
waitForContainer() {
|
||||
local NODE="$1"
|
||||
local CONTAINER="$2"
|
||||
local TIMEOUT="${3:-60}"
|
||||
|
||||
echo "Waiting for container $CONTAINER on $NODE (timeout: ${TIMEOUT}s)..."
|
||||
local elapsed=0
|
||||
local last_output=""
|
||||
while [ $elapsed -lt $TIMEOUT ]; do
|
||||
last_output=$(cmd "$NODE" "podman ps --filter name=$CONTAINER --format '{{.Status}}'" 2>&1)
|
||||
local status="$last_output"
|
||||
if [[ "$status" == *"Up"* ]]; then
|
||||
echo " ✓ Container $CONTAINER is running"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
|
||||
echo " ✗ Container $CONTAINER did not start within ${TIMEOUT}s"
|
||||
if [ -n "$last_output" ] && [[ "$last_output" == ERROR:* ]]; then
|
||||
echo " $last_output"
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
waitForService() {
|
||||
local NODE="$1"
|
||||
local SERVICE="$2"
|
||||
local TIMEOUT="${3:-60}"
|
||||
|
||||
echo "Waiting for service $SERVICE on $NODE (timeout: ${TIMEOUT}s)..."
|
||||
local elapsed=0
|
||||
local last_output=""
|
||||
while [ $elapsed -lt $TIMEOUT ]; do
|
||||
last_output=$(cmd "$NODE" "systemctl is-active $SERVICE" 2>&1)
|
||||
local status="$last_output"
|
||||
if [[ "$status" == *"active"* ]]; then
|
||||
echo " ✓ Service $SERVICE is active"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
|
||||
echo " ✗ Service $SERVICE did not become active within ${TIMEOUT}s"
|
||||
if [ -n "$last_output" ] && [[ "$last_output" == ERROR:* ]]; then
|
||||
echo " $last_output"
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
getContainerLogs() {
|
||||
local NODE="$1"
|
||||
local CONTAINER="$2"
|
||||
local LINES="${3:-50}"
|
||||
|
||||
echo "--- Container logs for $CONTAINER on $NODE ---"
|
||||
cmd "$NODE" "podman logs --tail $LINES $CONTAINER" 2>&1
|
||||
echo "--- End of logs ---"
|
||||
}
|
||||
|
||||
getServiceLogs() {
|
||||
local NODE="$1"
|
||||
local SERVICE="$2"
|
||||
local LINES="${3:-50}"
|
||||
|
||||
echo "--- Service logs for $SERVICE on $NODE ---"
|
||||
cmd "$NODE" "journalctl -n $LINES -u $SERVICE" 2>&1
|
||||
echo "--- End of logs ---"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Info output
|
||||
# ============================================================================
|
||||
|
||||
# Print info line (not pass/fail)
|
||||
# Usage: print_info "label" "value"
|
||||
print_info() {
|
||||
local label="$1"
|
||||
local value="$2"
|
||||
|
||||
echo -e " ${GREEN}✓${NC} $label: $value [info]"
|
||||
}
|
||||
|
||||
# Print cleanup success
|
||||
# Usage: print_cleanup "label"
|
||||
print_cleanup() {
|
||||
local label="$1"
|
||||
|
||||
echo -e " ${GREEN}✓${NC} $label [pass]"
|
||||
}
|
||||
Reference in New Issue
Block a user