From d836e4608ccd19968b5d8eb7a64523be3a3795f2 Mon Sep 17 00:00:00 2001 From: Ruben Hensen Date: Sun, 25 Oct 2026 03:48:05 +0100 Subject: [PATCH] gatus: check backups and Vault's seal from inside the cluster --- gatus/backup-check.yaml | 257 ++++++++++++++++++++++++++++++++++++ gatus/config.yaml | 33 +++++ gatus/gatus-deployment.yaml | 3 + gatus/kustomization.yaml | 1 + gatus/networkpolicy.yaml | 19 +++ 5 files changed, 313 insertions(+) create mode 100644 gatus/backup-check.yaml diff --git a/gatus/backup-check.yaml b/gatus/backup-check.yaml new file mode 100644 index 0000000..bc22ab3 --- /dev/null +++ b/gatus/backup-check.yaml @@ -0,0 +1,257 @@ +# Checks what Gatus can't see from outside and reports it to Gatus as +# external endpoints (group "cluster" on s2), which alert through ntfy when +# a check fails or when no report arrives (heartbeat). Velero's scheduled +# backups failed every night from August to 2026-10-21 without anyone +# noticing; a sealed Vault after a reboot looks fine from outside too. +# +# Every hour, from the Kubernetes API: +# - velero-daily / velero-weekly: the newest finished backup of the +# schedule is Completed and younger than 26h / 8 days +# - longhorn: every attached volume is healthy and has a completed backup +# younger than 26h (detached volumes are skipped; the recurring job skips +# them too) +# - vault-unsealed: all Vault pods report vault-sealed=false +# +# The token is generated by ESO and never leaves the cluster; Gatus reads +# it from the same Secret. +apiVersion: generators.external-secrets.io/v1alpha1 +kind: Password +metadata: + name: backup-check-token +spec: + length: 40 + digits: 10 + symbols: 0 + noUpper: false + allowRepeat: true + secretKeys: + - BACKUP_CHECK_TOKEN +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: backup-check-token +spec: + # Generate once; a new token would need a Gatus restart. + refreshInterval: "0" + target: + name: backup-check-token + creationPolicy: Owner + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: backup-check-token +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: backup-check +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: backup-check +rules: + - apiGroups: ["velero.io"] + resources: ["backups"] + verbs: ["list"] + - apiGroups: ["longhorn.io"] + resources: ["backups", "volumes"] + verbs: ["list"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: backup-check +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: backup-check +subjects: + - kind: ServiceAccount + name: backup-check + namespace: gatus +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-check +data: + check.py: | + import datetime as dt + import json + import os + import ssl + import sys + import urllib.parse + import urllib.request + + SA = "/var/run/secrets/kubernetes.io/serviceaccount" + API = "https://kubernetes.default.svc" + GATUS = os.environ.get("GATUS_URL", "http://gatus:8080") + TOKEN = os.environ["BACKUP_CHECK_TOKEN"] + NOW = dt.datetime.now(dt.timezone.utc) + FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"} + + with open(f"{SA}/token") as f: + sa_token = f.read() + api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt") + + + def items(path): + req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"}) + with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r: + return json.load(r)["items"] + + + def parse(ts): + return dt.datetime.fromisoformat(ts.replace("Z", "+00:00")) + + + def age(ts): + hours = (NOW - parse(ts)).total_seconds() / 3600 + return f"{hours:.0f}h" + + + def velero(schedule, max_hours): + backups = [ + b for b in items("/apis/velero.io/v1/namespaces/velero/backups") + if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule + and b.get("status", {}).get("phase") in FINISHED + ] + if not backups: + return f"no finished {schedule} backup" + newest = max(backups, key=lambda b: b["status"].get("startTimestamp", "")) + status = newest["status"] + name = newest["metadata"]["name"] + if status["phase"] != "Completed": + return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)" + done = status.get("completionTimestamp") + if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600: + return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old" + return None + + + def longhorn(max_hours): + newest = {} + for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"): + s = b.get("status", {}) + if s.get("state") == "Completed" and s.get("backupCreatedAt"): + v = s.get("volumeName") + newest[v] = max(newest.get(v, ""), s["backupCreatedAt"]) + problems = [] + for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"): + s = v.get("status", {}) + if s.get("state") != "attached": + continue + name = v["metadata"]["name"] + pvc = s.get("kubernetesStatus", {}).get("pvcName") or name + if s.get("robustness") != "healthy": + problems.append(f"{pvc} is {s.get('robustness')}") + last = newest.get(name) + if not last: + problems.append(f"{pvc} has no backup") + elif (NOW - parse(last)).total_seconds() > max_hours * 3600: + problems.append(f"{pvc} last backed up {age(last)} ago") + return "; ".join(problems) or None + + + def vault(): + pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver") + if not pods: + return "no Vault pods" + sealed = [p["metadata"]["name"] for p in pods + if p["metadata"].get("labels", {}).get("vault-sealed") != "false"] + return f"sealed: {', '.join(sealed)}" if sealed else None + + + def push(key, error): + query = {"success": "false" if error else "true"} + if error: + query["error"] = error[:500] + url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}" + req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"}) + with urllib.request.urlopen(req, timeout=30) as r: + r.read() + + + checks = { + "cluster_velero-daily": lambda: velero("daily-backup", 26), + "cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2), + "cluster_longhorn": lambda: longhorn(26), + "cluster_vault-unsealed": vault, + } + failed_pushes = 0 + for key, check in checks.items(): + try: + error = check() + except Exception as e: # report a broken check as a failure + error = f"check failed: {e}" + print(f"{key}: {error or 'ok'}", flush=True) + try: + push(key, error) + except Exception as e: + print(f"{key}: push to Gatus failed: {e}", flush=True) + failed_pushes += 1 + sys.exit(1 if failed_pushes else 0) +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-check +spec: + schedule: "17 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + app: backup-check + spec: + serviceAccountName: backup-check + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: check + image: python:3.13.16-alpine3.24 + command: ["python", "/script/check.py"] + env: + - name: PYTHONDONTWRITEBYTECODE + value: "1" + envFrom: + - secretRef: + name: backup-check-token + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: script + mountPath: /script + readOnly: true + volumes: + - name: script + configMap: + name: backup-check diff --git a/gatus/config.yaml b/gatus/config.yaml index 55447e9..6d8bc05 100644 --- a/gatus/config.yaml +++ b/gatus/config.yaml @@ -136,3 +136,36 @@ endpoints: - "[STATUS] == 200" - "[BODY].status == UP" alerts: *ntfy + +# Pushed every hour by the backup-check CronJob (backup-check.yaml), which +# looks at what can't be seen from outside. s2 alerts on these itself: s1 +# has no view into the cluster. +external-endpoints: + - name: velero-daily + group: cluster + token: ${BACKUP_CHECK_TOKEN} + heartbeat: + interval: 3h + alerts: &cluster-alert + - type: ntfy + failure-threshold: 2 + success-threshold: 1 + send-on-resolved: true + - name: velero-weekly + group: cluster + token: ${BACKUP_CHECK_TOKEN} + heartbeat: + interval: 3h + alerts: *cluster-alert + - name: longhorn + group: cluster + token: ${BACKUP_CHECK_TOKEN} + heartbeat: + interval: 3h + alerts: *cluster-alert + - name: vault-unsealed + group: cluster + token: ${BACKUP_CHECK_TOKEN} + heartbeat: + interval: 3h + alerts: *cluster-alert diff --git a/gatus/gatus-deployment.yaml b/gatus/gatus-deployment.yaml index 3e3e85e..a609a98 100644 --- a/gatus/gatus-deployment.yaml +++ b/gatus/gatus-deployment.yaml @@ -25,6 +25,9 @@ spec: envFrom: - secretRef: name: gatus-ntfy + # Token for the external endpoints, see backup-check.yaml + - secretRef: + name: backup-check-token ports: - containerPort: 8080 protocol: TCP diff --git a/gatus/kustomization.yaml b/gatus/kustomization.yaml index 8c8cbf9..d6ea3eb 100644 --- a/gatus/kustomization.yaml +++ b/gatus/kustomization.yaml @@ -11,6 +11,7 @@ resources: - certificate.yaml - httproute.yaml - networkpolicy.yaml + - backup-check.yaml configMapGenerator: - name: gatus-config diff --git a/gatus/networkpolicy.yaml b/gatus/networkpolicy.yaml index 6970182..1791dbf 100644 --- a/gatus/networkpolicy.yaml +++ b/gatus/networkpolicy.yaml @@ -64,3 +64,22 @@ spec: - 169.254.0.0/16 - 172.16.0.0/12 - 192.168.0.0/16 +--- +# backup-check.yaml reads backups, volumes and Vault's pods from the API +# server (the node, port 16443). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: backup-check-to-api +spec: + podSelector: + matchLabels: + app: backup-check + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 192.168.1.204/32 + ports: + - port: 16443