# Checks what Gatus can't see from outside and reports it to Gatus as # external endpoints (group "cluster" on s2), which alert through ntfy when # a check fails or when no report arrives (heartbeat). Velero's scheduled # backups failed every night from August to 2026-10-21 without anyone # noticing; a sealed Vault after a reboot looks fine from outside too. # # Every hour, from the Kubernetes API: # - velero-daily / velero-weekly: the newest finished backup of the # schedule is Completed and younger than 26h / 8 days # - longhorn: every attached volume is healthy and has a completed backup # younger than 26h (detached volumes are skipped; the recurring job skips # them too) # - vault-unsealed: all Vault pods report vault-sealed=false # - argocd-apps: every Argo CD application is Synced and Healthy # - postgres: every CloudNativePG cluster is in a healthy state # # The token is generated by ESO and never leaves the cluster; Gatus reads # it from the same Secret. apiVersion: generators.external-secrets.io/v1alpha1 kind: Password metadata: name: backup-check-token spec: length: 40 digits: 10 symbols: 0 noUpper: false allowRepeat: true secretKeys: - BACKUP_CHECK_TOKEN --- apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: backup-check-token spec: # Generate once; a new token would need a Gatus restart. refreshInterval: "0" target: name: backup-check-token creationPolicy: Owner dataFrom: - sourceRef: generatorRef: apiVersion: generators.external-secrets.io/v1alpha1 kind: Password name: backup-check-token --- apiVersion: v1 kind: ServiceAccount metadata: name: backup-check --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: backup-check rules: - apiGroups: ["velero.io"] resources: ["backups"] verbs: ["list"] - apiGroups: ["longhorn.io"] resources: ["backups", "volumes"] verbs: ["list"] - apiGroups: [""] resources: ["pods"] verbs: ["list"] - apiGroups: ["argoproj.io"] resources: ["applications"] verbs: ["list"] - apiGroups: ["postgresql.cnpg.io"] resources: ["clusters"] verbs: ["list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: backup-check roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: backup-check subjects: - kind: ServiceAccount name: backup-check namespace: gatus --- apiVersion: v1 kind: ConfigMap metadata: name: backup-check data: check.py: | import datetime as dt import json import os import ssl import sys import urllib.parse import urllib.request SA = "/var/run/secrets/kubernetes.io/serviceaccount" API = "https://kubernetes.default.svc" GATUS = os.environ.get("GATUS_URL", "http://gatus:8080") TOKEN = os.environ["BACKUP_CHECK_TOKEN"] NOW = dt.datetime.now(dt.timezone.utc) FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"} with open(f"{SA}/token") as f: sa_token = f.read() api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt") def items(path): req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"}) with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r: return json.load(r)["items"] def parse(ts): return dt.datetime.fromisoformat(ts.replace("Z", "+00:00")) def age(ts): hours = (NOW - parse(ts)).total_seconds() / 3600 return f"{hours:.0f}h" def velero(schedule, max_hours): backups = [ b for b in items("/apis/velero.io/v1/namespaces/velero/backups") if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule and b.get("status", {}).get("phase") in FINISHED ] if not backups: return f"no finished {schedule} backup" newest = max(backups, key=lambda b: b["status"].get("startTimestamp", "")) status = newest["status"] name = newest["metadata"]["name"] if status["phase"] != "Completed": return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)" done = status.get("completionTimestamp") if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600: return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old" return None def longhorn(max_hours): newest = {} for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"): s = b.get("status", {}) if s.get("state") == "Completed" and s.get("backupCreatedAt"): v = s.get("volumeName") newest[v] = max(newest.get(v, ""), s["backupCreatedAt"]) problems = [] for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"): s = v.get("status", {}) if s.get("state") != "attached": continue name = v["metadata"]["name"] pvc = s.get("kubernetesStatus", {}).get("pvcName") or name if s.get("robustness") != "healthy": problems.append(f"{pvc} is {s.get('robustness')}") last = newest.get(name) if not last: problems.append(f"{pvc} has no backup") elif (NOW - parse(last)).total_seconds() > max_hours * 3600: problems.append(f"{pvc} last backed up {age(last)} ago") return "; ".join(problems) or None def vault(): pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver") if not pods: return "no Vault pods" sealed = [p["metadata"]["name"] for p in pods if p["metadata"].get("labels", {}).get("vault-sealed") != "false"] return f"sealed: {', '.join(sealed)}" if sealed else None def argocd(): bad = [] for a in items("/apis/argoproj.io/v1alpha1/namespaces/argocd/applications"): s = a.get("status", {}) sync = s.get("sync", {}).get("status") health = s.get("health", {}).get("status") if sync != "Synced" or health != "Healthy": bad.append(f"{a['metadata']['name']} {sync}/{health}") return "; ".join(bad) or None def postgres(): bad = [f"{c['metadata']['namespace']}/{c['metadata']['name']}: {c.get('status', {}).get('phase')}" for c in items("/apis/postgresql.cnpg.io/v1/clusters") if c.get("status", {}).get("phase") != "Cluster in healthy state"] return "; ".join(bad) or None def push(key, error): query = {"success": "false" if error else "true"} if error: query["error"] = error[:500] url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}" req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"}) with urllib.request.urlopen(req, timeout=30) as r: r.read() checks = { "cluster_velero-daily": lambda: velero("daily-backup", 26), "cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2), "cluster_longhorn": lambda: longhorn(26), "cluster_vault-unsealed": vault, "cluster_argocd-apps": argocd, "cluster_postgres": postgres, } failed_pushes = 0 for key, check in checks.items(): try: error = check() except Exception as e: # report a broken check as a failure error = f"check failed: {e}" print(f"{key}: {error or 'ok'}", flush=True) try: push(key, error) except Exception as e: print(f"{key}: push to Gatus failed: {e}", flush=True) failed_pushes += 1 sys.exit(1 if failed_pushes else 0) --- apiVersion: batch/v1 kind: CronJob metadata: name: backup-check spec: schedule: "17 * * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 3 jobTemplate: spec: backoffLimit: 1 activeDeadlineSeconds: 300 template: metadata: labels: app: backup-check spec: serviceAccountName: backup-check restartPolicy: Never securityContext: runAsNonRoot: true runAsUser: 65534 runAsGroup: 65534 seccompProfile: type: RuntimeDefault containers: - name: check image: python:3.15.2-alpine3.24 command: ["python", "/script/check.py"] env: - name: PYTHONDONTWRITEBYTECODE value: "1" envFrom: - secretRef: name: backup-check-token resources: requests: cpu: 10m memory: 32Mi limits: memory: 128Mi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: drop: ["ALL"] volumeMounts: - name: script mountPath: /script readOnly: true volumes: - name: script configMap: name: backup-check