gatus: check backups and Vault's seal from inside the cluster

gatus/backup-check.yaml

@@ -0,0 +1,257 @@
+# Checks what Gatus can't see from outside and reports it to Gatus as
+# external endpoints (group "cluster" on s2), which alert through ntfy when
+# a check fails or when no report arrives (heartbeat). Velero's scheduled
+# backups failed every night from August to 2026-10-21 without anyone
+# noticing; a sealed Vault after a reboot looks fine from outside too.
+#
+# Every hour, from the Kubernetes API:
+# - velero-daily / velero-weekly: the newest finished backup of the
+#   schedule is Completed and younger than 26h / 8 days
+# - longhorn: every attached volume is healthy and has a completed backup
+#   younger than 26h (detached volumes are skipped; the recurring job skips
+#   them too)
+# - vault-unsealed: all Vault pods report vault-sealed=false
+#
+# The token is generated by ESO and never leaves the cluster; Gatus reads
+# it from the same Secret.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: backup-check-token
+spec:
+  length: 40
+  digits: 10
+  symbols: 0
+  noUpper: false
+  allowRepeat: true
+  secretKeys:
+    - BACKUP_CHECK_TOKEN
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: backup-check-token
+spec:
+  # Generate once; a new token would need a Gatus restart.
+  refreshInterval: "0"
+  target:
+    name: backup-check-token
+    creationPolicy: Owner
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: backup-check-token
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: backup-check
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: backup-check
+rules:
+  - apiGroups: ["velero.io"]
+    resources: ["backups"]
+    verbs: ["list"]
+  - apiGroups: ["longhorn.io"]
+    resources: ["backups", "volumes"]
+    verbs: ["list"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: backup-check
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: backup-check
+subjects:
+  - kind: ServiceAccount
+    name: backup-check
+    namespace: gatus
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: backup-check
+data:
+  check.py: |
+    import datetime as dt
+    import json
+    import os
+    import ssl
+    import sys
+    import urllib.parse
+    import urllib.request
+
+    SA = "/var/run/secrets/kubernetes.io/serviceaccount"
+    API = "https://kubernetes.default.svc"
+    GATUS = os.environ.get("GATUS_URL", "http://gatus:8080")
+    TOKEN = os.environ["BACKUP_CHECK_TOKEN"]
+    NOW = dt.datetime.now(dt.timezone.utc)
+    FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"}
+
+    with open(f"{SA}/token") as f:
+        sa_token = f.read()
+    api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt")
+
+
+    def items(path):
+        req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"})
+        with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r:
+            return json.load(r)["items"]
+
+
+    def parse(ts):
+        return dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
+
+
+    def age(ts):
+        hours = (NOW - parse(ts)).total_seconds() / 3600
+        return f"{hours:.0f}h"
+
+
+    def velero(schedule, max_hours):
+        backups = [
+            b for b in items("/apis/velero.io/v1/namespaces/velero/backups")
+            if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule
+            and b.get("status", {}).get("phase") in FINISHED
+        ]
+        if not backups:
+            return f"no finished {schedule} backup"
+        newest = max(backups, key=lambda b: b["status"].get("startTimestamp", ""))
+        status = newest["status"]
+        name = newest["metadata"]["name"]
+        if status["phase"] != "Completed":
+            return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)"
+        done = status.get("completionTimestamp")
+        if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600:
+            return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old"
+        return None
+
+
+    def longhorn(max_hours):
+        newest = {}
+        for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"):
+            s = b.get("status", {})
+            if s.get("state") == "Completed" and s.get("backupCreatedAt"):
+                v = s.get("volumeName")
+                newest[v] = max(newest.get(v, ""), s["backupCreatedAt"])
+        problems = []
+        for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"):
+            s = v.get("status", {})
+            if s.get("state") != "attached":
+                continue
+            name = v["metadata"]["name"]
+            pvc = s.get("kubernetesStatus", {}).get("pvcName") or name
+            if s.get("robustness") != "healthy":
+                problems.append(f"{pvc} is {s.get('robustness')}")
+            last = newest.get(name)
+            if not last:
+                problems.append(f"{pvc} has no backup")
+            elif (NOW - parse(last)).total_seconds() > max_hours * 3600:
+                problems.append(f"{pvc} last backed up {age(last)} ago")
+        return "; ".join(problems) or None
+
+
+    def vault():
+        pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver")
+        if not pods:
+            return "no Vault pods"
+        sealed = [p["metadata"]["name"] for p in pods
+                  if p["metadata"].get("labels", {}).get("vault-sealed") != "false"]
+        return f"sealed: {', '.join(sealed)}" if sealed else None
+
+
+    def push(key, error):
+        query = {"success": "false" if error else "true"}
+        if error:
+            query["error"] = error[:500]
+        url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}"
+        req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"})
+        with urllib.request.urlopen(req, timeout=30) as r:
+            r.read()
+
+
+    checks = {
+        "cluster_velero-daily": lambda: velero("daily-backup", 26),
+        "cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
+        "cluster_longhorn": lambda: longhorn(26),
+        "cluster_vault-unsealed": vault,
+    }
+    failed_pushes = 0
+    for key, check in checks.items():
+        try:
+            error = check()
+        except Exception as e:  # report a broken check as a failure
+            error = f"check failed: {e}"
+        print(f"{key}: {error or 'ok'}", flush=True)
+        try:
+            push(key, error)
+        except Exception as e:
+            print(f"{key}: push to Gatus failed: {e}", flush=True)
+            failed_pushes += 1
+    sys.exit(1 if failed_pushes else 0)
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: backup-check
+spec:
+  schedule: "17 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 300
+      template:
+        metadata:
+          labels:
+            app: backup-check
+        spec:
+          serviceAccountName: backup-check
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: check
+              image: python:3.13.16-alpine3.24
+              command: ["python", "/script/check.py"]
+              env:
+                - name: PYTHONDONTWRITEBYTECODE
+                  value: "1"
+              envFrom:
+                - secretRef:
+                    name: backup-check-token
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 32Mi
+                limits:
+                  memory: 128Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: script
+                  mountPath: /script
+                  readOnly: true
+          volumes:
+            - name: script
+              configMap:
+                name: backup-check

gatus/config.yaml

@@ -136,3 +136,36 @@ endpoints:
       - "[STATUS] == 200"
       - "[BODY].status == UP"
     alerts: *ntfy
+
+# Pushed every hour by the backup-check CronJob (backup-check.yaml), which
+# looks at what can't be seen from outside. s2 alerts on these itself: s1
+# has no view into the cluster.
+external-endpoints:
+  - name: velero-daily
+    group: cluster
+    token: ${BACKUP_CHECK_TOKEN}
+    heartbeat:
+      interval: 3h
+    alerts: &cluster-alert
+      - type: ntfy
+        failure-threshold: 2
+        success-threshold: 1
+        send-on-resolved: true
+  - name: velero-weekly
+    group: cluster
+    token: ${BACKUP_CHECK_TOKEN}
+    heartbeat:
+      interval: 3h
+    alerts: *cluster-alert
+  - name: longhorn
+    group: cluster
+    token: ${BACKUP_CHECK_TOKEN}
+    heartbeat:
+      interval: 3h
+    alerts: *cluster-alert
+  - name: vault-unsealed
+    group: cluster
+    token: ${BACKUP_CHECK_TOKEN}
+    heartbeat:
+      interval: 3h
+    alerts: *cluster-alert

gatus/gatus-deployment.yaml

@@ -25,6 +25,9 @@ spec:
           envFrom:
             - secretRef:
                 name: gatus-ntfy
+            # Token for the external endpoints, see backup-check.yaml
+            - secretRef:
+                name: backup-check-token
           ports:
             - containerPort: 8080
               protocol: TCP

gatus/kustomization.yaml

@@ -11,6 +11,7 @@ resources:
   - certificate.yaml
   - httproute.yaml
   - networkpolicy.yaml
+  - backup-check.yaml
 
 configMapGenerator:
   - name: gatus-config

gatus/networkpolicy.yaml

@@ -64,3 +64,22 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# backup-check.yaml reads backups, volumes and Vault's pods from the API
+# server (the node, port 16443).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: backup-check-to-api
+spec:
+  podSelector:
+    matchLabels:
+      app: backup-check
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 192.168.1.204/32
+      ports:
+        - port: 16443