mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-15 22:29:09 +01:00
286 lines
9.3 KiB
YAML
286 lines
9.3 KiB
YAML
# Checks what Gatus can't see from outside and reports it to Gatus as
|
|
# external endpoints (group "cluster" on s2), which alert through ntfy when
|
|
# a check fails or when no report arrives (heartbeat). Velero's scheduled
|
|
# backups failed every night from August to 2026-10-21 without anyone
|
|
# noticing; a sealed Vault after a reboot looks fine from outside too.
|
|
#
|
|
# Every hour, from the Kubernetes API:
|
|
# - velero-daily / velero-weekly: the newest finished backup of the
|
|
# schedule is Completed and younger than 26h / 8 days
|
|
# - longhorn: every attached volume is healthy and has a completed backup
|
|
# younger than 26h (detached volumes are skipped; the recurring job skips
|
|
# them too)
|
|
# - vault-unsealed: all Vault pods report vault-sealed=false
|
|
# - argocd-apps: every Argo CD application is Synced and Healthy
|
|
# - postgres: every CloudNativePG cluster is in a healthy state
|
|
#
|
|
# The token is generated by ESO and never leaves the cluster; Gatus reads
|
|
# it from the same Secret.
|
|
apiVersion: generators.external-secrets.io/v1alpha1
|
|
kind: Password
|
|
metadata:
|
|
name: backup-check-token
|
|
spec:
|
|
length: 40
|
|
digits: 10
|
|
symbols: 0
|
|
noUpper: false
|
|
allowRepeat: true
|
|
secretKeys:
|
|
- BACKUP_CHECK_TOKEN
|
|
---
|
|
apiVersion: external-secrets.io/v1
|
|
kind: ExternalSecret
|
|
metadata:
|
|
name: backup-check-token
|
|
spec:
|
|
# Generate once; a new token would need a Gatus restart.
|
|
refreshInterval: "0"
|
|
target:
|
|
name: backup-check-token
|
|
creationPolicy: Owner
|
|
dataFrom:
|
|
- sourceRef:
|
|
generatorRef:
|
|
apiVersion: generators.external-secrets.io/v1alpha1
|
|
kind: Password
|
|
name: backup-check-token
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: backup-check
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: backup-check
|
|
rules:
|
|
- apiGroups: ["velero.io"]
|
|
resources: ["backups"]
|
|
verbs: ["list"]
|
|
- apiGroups: ["longhorn.io"]
|
|
resources: ["backups", "volumes"]
|
|
verbs: ["list"]
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["list"]
|
|
- apiGroups: ["argoproj.io"]
|
|
resources: ["applications"]
|
|
verbs: ["list"]
|
|
- apiGroups: ["postgresql.cnpg.io"]
|
|
resources: ["clusters"]
|
|
verbs: ["list"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: backup-check
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: backup-check
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: backup-check
|
|
namespace: gatus
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: backup-check
|
|
data:
|
|
check.py: |
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import ssl
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
SA = "/var/run/secrets/kubernetes.io/serviceaccount"
|
|
API = "https://kubernetes.default.svc"
|
|
GATUS = os.environ.get("GATUS_URL", "http://gatus:8080")
|
|
TOKEN = os.environ["BACKUP_CHECK_TOKEN"]
|
|
NOW = dt.datetime.now(dt.timezone.utc)
|
|
FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"}
|
|
|
|
with open(f"{SA}/token") as f:
|
|
sa_token = f.read()
|
|
api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt")
|
|
|
|
|
|
def items(path):
|
|
req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"})
|
|
with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r:
|
|
return json.load(r)["items"]
|
|
|
|
|
|
def parse(ts):
|
|
return dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
|
|
|
|
def age(ts):
|
|
hours = (NOW - parse(ts)).total_seconds() / 3600
|
|
return f"{hours:.0f}h"
|
|
|
|
|
|
def velero(schedule, max_hours):
|
|
backups = [
|
|
b for b in items("/apis/velero.io/v1/namespaces/velero/backups")
|
|
if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule
|
|
and b.get("status", {}).get("phase") in FINISHED
|
|
]
|
|
if not backups:
|
|
return f"no finished {schedule} backup"
|
|
newest = max(backups, key=lambda b: b["status"].get("startTimestamp", ""))
|
|
status = newest["status"]
|
|
name = newest["metadata"]["name"]
|
|
if status["phase"] != "Completed":
|
|
return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)"
|
|
done = status.get("completionTimestamp")
|
|
if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600:
|
|
return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old"
|
|
return None
|
|
|
|
|
|
def longhorn(max_hours):
|
|
newest = {}
|
|
for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"):
|
|
s = b.get("status", {})
|
|
if s.get("state") == "Completed" and s.get("backupCreatedAt"):
|
|
v = s.get("volumeName")
|
|
newest[v] = max(newest.get(v, ""), s["backupCreatedAt"])
|
|
problems = []
|
|
for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"):
|
|
s = v.get("status", {})
|
|
if s.get("state") != "attached":
|
|
continue
|
|
name = v["metadata"]["name"]
|
|
pvc = s.get("kubernetesStatus", {}).get("pvcName") or name
|
|
if s.get("robustness") != "healthy":
|
|
problems.append(f"{pvc} is {s.get('robustness')}")
|
|
last = newest.get(name)
|
|
if not last:
|
|
problems.append(f"{pvc} has no backup")
|
|
elif (NOW - parse(last)).total_seconds() > max_hours * 3600:
|
|
problems.append(f"{pvc} last backed up {age(last)} ago")
|
|
return "; ".join(problems) or None
|
|
|
|
|
|
def vault():
|
|
pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver")
|
|
if not pods:
|
|
return "no Vault pods"
|
|
sealed = [p["metadata"]["name"] for p in pods
|
|
if p["metadata"].get("labels", {}).get("vault-sealed") != "false"]
|
|
return f"sealed: {', '.join(sealed)}" if sealed else None
|
|
|
|
|
|
def argocd():
|
|
bad = []
|
|
for a in items("/apis/argoproj.io/v1alpha1/namespaces/argocd/applications"):
|
|
s = a.get("status", {})
|
|
sync = s.get("sync", {}).get("status")
|
|
health = s.get("health", {}).get("status")
|
|
if sync != "Synced" or health != "Healthy":
|
|
bad.append(f"{a['metadata']['name']} {sync}/{health}")
|
|
return "; ".join(bad) or None
|
|
|
|
|
|
def postgres():
|
|
bad = [f"{c['metadata']['namespace']}/{c['metadata']['name']}: {c.get('status', {}).get('phase')}"
|
|
for c in items("/apis/postgresql.cnpg.io/v1/clusters")
|
|
if c.get("status", {}).get("phase") != "Cluster in healthy state"]
|
|
return "; ".join(bad) or None
|
|
|
|
|
|
def push(key, error):
|
|
query = {"success": "false" if error else "true"}
|
|
if error:
|
|
query["error"] = error[:500]
|
|
url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}"
|
|
req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"})
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
r.read()
|
|
|
|
|
|
checks = {
|
|
"cluster_velero-daily": lambda: velero("daily-backup", 26),
|
|
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
|
|
"cluster_longhorn": lambda: longhorn(26),
|
|
"cluster_vault-unsealed": vault,
|
|
"cluster_argocd-apps": argocd,
|
|
"cluster_postgres": postgres,
|
|
}
|
|
failed_pushes = 0
|
|
for key, check in checks.items():
|
|
try:
|
|
error = check()
|
|
except Exception as e: # report a broken check as a failure
|
|
error = f"check failed: {e}"
|
|
print(f"{key}: {error or 'ok'}", flush=True)
|
|
try:
|
|
push(key, error)
|
|
except Exception as e:
|
|
print(f"{key}: push to Gatus failed: {e}", flush=True)
|
|
failed_pushes += 1
|
|
sys.exit(1 if failed_pushes else 0)
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: backup-check
|
|
spec:
|
|
schedule: "17 * * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 1
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 1
|
|
activeDeadlineSeconds: 300
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: backup-check
|
|
spec:
|
|
serviceAccountName: backup-check
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: check
|
|
image: python:3.15.1-alpine3.24
|
|
command: ["python", "/script/check.py"]
|
|
env:
|
|
- name: PYTHONDONTWRITEBYTECODE
|
|
value: "1"
|
|
envFrom:
|
|
- secretRef:
|
|
name: backup-check-token
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 128Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumeMounts:
|
|
- name: script
|
|
mountPath: /script
|
|
readOnly: true
|
|
volumes:
|
|
- name: script
|
|
configMap:
|
|
name: backup-check
|