mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 06:29:09 +01:00
gatus: check backups and Vault's seal from inside the cluster
This commit is contained in:
1 parent
1f56e67e8b
commit
d836e4608c
5 files changed
+313
No files matched your search
@@ -0,0 +1,257 @@
|
|||||||
|
# Checks what Gatus can't see from outside and reports it to Gatus as
|
||||||
|
# external endpoints (group "cluster" on s2), which alert through ntfy when
|
||||||
|
# a check fails or when no report arrives (heartbeat). Velero's scheduled
|
||||||
|
# backups failed every night from August to 2026-10-21 without anyone
|
||||||
|
# noticing; a sealed Vault after a reboot looks fine from outside too.
|
||||||
|
#
|
||||||
|
# Every hour, from the Kubernetes API:
|
||||||
|
# - velero-daily / velero-weekly: the newest finished backup of the
|
||||||
|
# schedule is Completed and younger than 26h / 8 days
|
||||||
|
# - longhorn: every attached volume is healthy and has a completed backup
|
||||||
|
# younger than 26h (detached volumes are skipped; the recurring job skips
|
||||||
|
# them too)
|
||||||
|
# - vault-unsealed: all Vault pods report vault-sealed=false
|
||||||
|
#
|
||||||
|
# The token is generated by ESO and never leaves the cluster; Gatus reads
|
||||||
|
# it from the same Secret.
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
metadata:
|
||||||
|
name: backup-check-token
|
||||||
|
spec:
|
||||||
|
length: 40
|
||||||
|
digits: 10
|
||||||
|
symbols: 0
|
||||||
|
noUpper: false
|
||||||
|
allowRepeat: true
|
||||||
|
secretKeys:
|
||||||
|
- BACKUP_CHECK_TOKEN
|
||||||
|
---
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: backup-check-token
|
||||||
|
spec:
|
||||||
|
# Generate once; a new token would need a Gatus restart.
|
||||||
|
refreshInterval: "0"
|
||||||
|
target:
|
||||||
|
name: backup-check-token
|
||||||
|
creationPolicy: Owner
|
||||||
|
dataFrom:
|
||||||
|
- sourceRef:
|
||||||
|
generatorRef:
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
name: backup-check-token
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
rules:
|
||||||
|
- apiGroups: ["velero.io"]
|
||||||
|
resources: ["backups"]
|
||||||
|
verbs: ["list"]
|
||||||
|
- apiGroups: ["longhorn.io"]
|
||||||
|
resources: ["backups", "volumes"]
|
||||||
|
verbs: ["list"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods"]
|
||||||
|
verbs: ["list"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: backup-check
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: backup-check
|
||||||
|
namespace: gatus
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
data:
|
||||||
|
check.py: |
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
SA = "/var/run/secrets/kubernetes.io/serviceaccount"
|
||||||
|
API = "https://kubernetes.default.svc"
|
||||||
|
GATUS = os.environ.get("GATUS_URL", "http://gatus:8080")
|
||||||
|
TOKEN = os.environ["BACKUP_CHECK_TOKEN"]
|
||||||
|
NOW = dt.datetime.now(dt.timezone.utc)
|
||||||
|
FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"}
|
||||||
|
|
||||||
|
with open(f"{SA}/token") as f:
|
||||||
|
sa_token = f.read()
|
||||||
|
api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt")
|
||||||
|
|
||||||
|
|
||||||
|
def items(path):
|
||||||
|
req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"})
|
||||||
|
with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r:
|
||||||
|
return json.load(r)["items"]
|
||||||
|
|
||||||
|
|
||||||
|
def parse(ts):
|
||||||
|
return dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||||
|
|
||||||
|
|
||||||
|
def age(ts):
|
||||||
|
hours = (NOW - parse(ts)).total_seconds() / 3600
|
||||||
|
return f"{hours:.0f}h"
|
||||||
|
|
||||||
|
|
||||||
|
def velero(schedule, max_hours):
|
||||||
|
backups = [
|
||||||
|
b for b in items("/apis/velero.io/v1/namespaces/velero/backups")
|
||||||
|
if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule
|
||||||
|
and b.get("status", {}).get("phase") in FINISHED
|
||||||
|
]
|
||||||
|
if not backups:
|
||||||
|
return f"no finished {schedule} backup"
|
||||||
|
newest = max(backups, key=lambda b: b["status"].get("startTimestamp", ""))
|
||||||
|
status = newest["status"]
|
||||||
|
name = newest["metadata"]["name"]
|
||||||
|
if status["phase"] != "Completed":
|
||||||
|
return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)"
|
||||||
|
done = status.get("completionTimestamp")
|
||||||
|
if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600:
|
||||||
|
return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def longhorn(max_hours):
|
||||||
|
newest = {}
|
||||||
|
for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"):
|
||||||
|
s = b.get("status", {})
|
||||||
|
if s.get("state") == "Completed" and s.get("backupCreatedAt"):
|
||||||
|
v = s.get("volumeName")
|
||||||
|
newest[v] = max(newest.get(v, ""), s["backupCreatedAt"])
|
||||||
|
problems = []
|
||||||
|
for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"):
|
||||||
|
s = v.get("status", {})
|
||||||
|
if s.get("state") != "attached":
|
||||||
|
continue
|
||||||
|
name = v["metadata"]["name"]
|
||||||
|
pvc = s.get("kubernetesStatus", {}).get("pvcName") or name
|
||||||
|
if s.get("robustness") != "healthy":
|
||||||
|
problems.append(f"{pvc} is {s.get('robustness')}")
|
||||||
|
last = newest.get(name)
|
||||||
|
if not last:
|
||||||
|
problems.append(f"{pvc} has no backup")
|
||||||
|
elif (NOW - parse(last)).total_seconds() > max_hours * 3600:
|
||||||
|
problems.append(f"{pvc} last backed up {age(last)} ago")
|
||||||
|
return "; ".join(problems) or None
|
||||||
|
|
||||||
|
|
||||||
|
def vault():
|
||||||
|
pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver")
|
||||||
|
if not pods:
|
||||||
|
return "no Vault pods"
|
||||||
|
sealed = [p["metadata"]["name"] for p in pods
|
||||||
|
if p["metadata"].get("labels", {}).get("vault-sealed") != "false"]
|
||||||
|
return f"sealed: {', '.join(sealed)}" if sealed else None
|
||||||
|
|
||||||
|
|
||||||
|
def push(key, error):
|
||||||
|
query = {"success": "false" if error else "true"}
|
||||||
|
if error:
|
||||||
|
query["error"] = error[:500]
|
||||||
|
url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}"
|
||||||
|
req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"})
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
r.read()
|
||||||
|
|
||||||
|
|
||||||
|
checks = {
|
||||||
|
"cluster_velero-daily": lambda: velero("daily-backup", 26),
|
||||||
|
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
|
||||||
|
"cluster_longhorn": lambda: longhorn(26),
|
||||||
|
"cluster_vault-unsealed": vault,
|
||||||
|
}
|
||||||
|
failed_pushes = 0
|
||||||
|
for key, check in checks.items():
|
||||||
|
try:
|
||||||
|
error = check()
|
||||||
|
except Exception as e: # report a broken check as a failure
|
||||||
|
error = f"check failed: {e}"
|
||||||
|
print(f"{key}: {error or 'ok'}", flush=True)
|
||||||
|
try:
|
||||||
|
push(key, error)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"{key}: push to Gatus failed: {e}", flush=True)
|
||||||
|
failed_pushes += 1
|
||||||
|
sys.exit(1 if failed_pushes else 0)
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
spec:
|
||||||
|
schedule: "17 * * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
activeDeadlineSeconds: 300
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: backup-check
|
||||||
|
spec:
|
||||||
|
serviceAccountName: backup-check
|
||||||
|
restartPolicy: Never
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 65534
|
||||||
|
runAsGroup: 65534
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: check
|
||||||
|
image: python:3.13.16-alpine3.24
|
||||||
|
command: ["python", "/script/check.py"]
|
||||||
|
env:
|
||||||
|
- name: PYTHONDONTWRITEBYTECODE
|
||||||
|
value: "1"
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: backup-check-token
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /script
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: backup-check
|
||||||
@@ -136,3 +136,36 @@ endpoints:
|
|||||||
- "[STATUS] == 200"
|
- "[STATUS] == 200"
|
||||||
- "[BODY].status == UP"
|
- "[BODY].status == UP"
|
||||||
alerts: *ntfy
|
alerts: *ntfy
|
||||||
|
|
||||||
|
# Pushed every hour by the backup-check CronJob (backup-check.yaml), which
|
||||||
|
# looks at what can't be seen from outside. s2 alerts on these itself: s1
|
||||||
|
# has no view into the cluster.
|
||||||
|
external-endpoints:
|
||||||
|
- name: velero-daily
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: &cluster-alert
|
||||||
|
- type: ntfy
|
||||||
|
failure-threshold: 2
|
||||||
|
success-threshold: 1
|
||||||
|
send-on-resolved: true
|
||||||
|
- name: velero-weekly
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: *cluster-alert
|
||||||
|
- name: longhorn
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: *cluster-alert
|
||||||
|
- name: vault-unsealed
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: *cluster-alert
|
||||||
@@ -25,6 +25,9 @@ spec:
|
|||||||
envFrom:
|
envFrom:
|
||||||
- secretRef:
|
- secretRef:
|
||||||
name: gatus-ntfy
|
name: gatus-ntfy
|
||||||
|
# Token for the external endpoints, see backup-check.yaml
|
||||||
|
- secretRef:
|
||||||
|
name: backup-check-token
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8080
|
- containerPort: 8080
|
||||||
protocol: TCP
|
protocol: TCP
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ resources:
|
|||||||
- certificate.yaml
|
- certificate.yaml
|
||||||
- httproute.yaml
|
- httproute.yaml
|
||||||
- networkpolicy.yaml
|
- networkpolicy.yaml
|
||||||
|
- backup-check.yaml
|
||||||
|
|
||||||
configMapGenerator:
|
configMapGenerator:
|
||||||
- name: gatus-config
|
- name: gatus-config
|
||||||
|
|||||||
@@ -64,3 +64,22 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# backup-check.yaml reads backups, volumes and Vault's pods from the API
|
||||||
|
# server (the node, port 16443).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: backup-check-to-api
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: backup-check
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
Reference in new issue
Block a user