From 3ffb10b6274cc167ea608c472da04a70415bcb16 Mon Sep 17 00:00:00 2001 From: Ruben Hensen Date: Tue, 20 Oct 2026 11:03:58 +0200 Subject: [PATCH] gatus: add s2.rubenhensen.nl status page on the home cluster Gatus with sqlite history, checking every public service plus its twin s1.rubenhensen.nl on Hetzner. Replaces the GitHub-hosted Upptime page. DNS gets s1 (Hetzner) and s2. s2 points straight at the home IP rather than through the Hetzner tunnel, so it stays reachable when Hetzner is down. gatus/README.md explains how to read the two pages and writes down the plan for a single self-hosted failover URL (PowerDNS LUA records). --- apps/templates/gatus-folder.yaml | 21 +++++ dns/domains/rubenhensen.nl.yaml | 10 ++ gatus/README.md | 79 ++++++++++++++++ gatus/config.yaml | 126 ++++++++++++++++++++++++++ gatus/data-persistentvolumeclaim.yaml | 10 ++ gatus/gatus-deployment.yaml | 54 +++++++++++ gatus/gatus-service.yaml | 11 +++ gatus/ingress.yaml | 25 +++++ gatus/kustomization.yaml | 15 +++ renovate.json | 3 +- 10 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 apps/templates/gatus-folder.yaml create mode 100644 gatus/README.md create mode 100644 gatus/config.yaml create mode 100644 gatus/data-persistentvolumeclaim.yaml create mode 100644 gatus/gatus-deployment.yaml create mode 100644 gatus/gatus-service.yaml create mode 100644 gatus/ingress.yaml create mode 100644 gatus/kustomization.yaml diff --git a/apps/templates/gatus-folder.yaml b/apps/templates/gatus-folder.yaml new file mode 100644 index 0000000..be630ef --- /dev/null +++ b/apps/templates/gatus-folder.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: gatus-folder + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/rubenhensen/k8scd.git + targetRevision: HEAD + path: gatus + destination: + server: https://kubernetes.default.svc + namespace: gatus + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true diff --git a/dns/domains/rubenhensen.nl.yaml b/dns/domains/rubenhensen.nl.yaml index c134432..59809ab 100644 --- a/dns/domains/rubenhensen.nl.yaml +++ b/dns/domains/rubenhensen.nl.yaml @@ -63,6 +63,16 @@ records: expire: 300 type: A content: "46.224.26.65" + # Gatus status pages. s2 points straight at home, not through the + # Hetzner tunnel, so it stays reachable when Hetzner is down. + - name: "s1" + expire: 300 + type: A + content: "46.224.26.65" + - name: "s2" + expire: 300 + type: A + content: "62.41.86.27" - name: "@" expire: 300 type: MX diff --git a/gatus/README.md b/gatus/README.md new file mode 100644 index 0000000..62dc78c --- /dev/null +++ b/gatus/README.md @@ -0,0 +1,79 @@ +# Gatus + +Two Gatus instances check the same endpoints from two places: + +| Status page | Runs on | Config | +|---|---|---| +| https://s1.rubenhensen.nl | Hetzner (node001) | `nix-infra-machine/nodes/node001.nix` | +| https://s2.rubenhensen.nl | Home cluster | `gatus/config.yaml` (this folder) | + +Each one also checks the other's `/health`. The s2 DNS record points straight +at the home IP instead of going through the Hetzner tunnel, so s2 stays +reachable when Hetzner is down. + +The endpoint list exists twice, so a new service has to be added in both +places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up +there automatically. s2 needs them added by hand. + +## Reading the two pages + +- s1 sees the homelab from outside, so trust it for homelab services. s2 sees + Hetzner from outside, so trust it for mail and anything else on Hetzner. +- Every `*.rubenhensen.nl` app is tunneled Hetzner → home, so a Hetzner + outage shows all of them as down on both pages. That is accurate: nobody + can reach them then. +- If s1 and s2 disagree about an endpoint, the problem is the network path + between one watcher and that service, not the service itself. +- Neither page has history for the time its own node was down. + +## Later: one URL with self-hosted DNS failover + +This is the plan for replacing s1/s2 with a single `status.rubenhensen.nl` +that always points at a watcher that's up, with no third party involved. + +Hand `status.rubenhensen.nl` over to two nameservers we run ourselves, one per +node, both PowerDNS Authoritative. PowerDNS LUA records health-check +addresses and only return the healthy ones: + +``` +@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.27'}})" +``` + +The double braces make two groups. PowerDNS returns the first group that has +a healthy address, so Hetzner is primary and home is the fallback. Each +nameserver runs the check from its own location. + +| Situation | Hetzner NS | Home NS | Visitors end up on | +|---|---|---|---| +| Both up | answers Hetzner | answers Hetzner | Hetzner | +| Hetzner down | dead | Hetzner check fails, answers home | home | +| Home down / home internet down | answers Hetzner | dead | Hetzner | +| Hetzner up, its Gatus broken | own check fails, answers home | answers home | home | + +A node that goes down takes its nameserver down with it, so DNS resolvers +switch to the other nameserver. The two nodes never have to agree with each +other: the answer depends on what the internet can reach. The 30s TTL caps +how long clients keep a stale answer. + +Steps: + +1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add + `status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`, + plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.27). + First check that TransIP accepts NS records for a subdomain. +2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and + the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp. +3. Home: run PowerDNS in the cluster with the same zone file, on 53/udp and + 53/tcp of the home IP (router port forward, and check how the cluster + exposes UDP). +4. Both Gatus instances also serve `status.rubenhensen.nl`. On Hetzner, add the + name to the SNI map, the same way as s1. +5. TLS: Hetzner gets its cert through HTTP-01 as usual. On Hetzner's :80 vhost + for `status`, serve local ACME challenges and proxy the rest to the home + cluster (`try_files $uri @home`). That way cert-manager's HTTP-01 also + works while DNS points at Hetzner. When Hetzner is down, DNS points home + and cert-manager reaches the home cluster directly. +6. Requirements: a static home IP (`homeIP` in node001.nix already assumes + one), and port 53 open on both nodes. + +After that, s1/s2 can be removed, or kept as direct links to each instance. diff --git a/gatus/config.yaml b/gatus/config.yaml new file mode 100644 index 0000000..d0712bc --- /dev/null +++ b/gatus/config.yaml @@ -0,0 +1,126 @@ +# s2.rubenhensen.nl: Gatus on the home cluster. +# Its twin s1.rubenhensen.nl runs on the Hetzner box, see +# nix-infra-machine/nodes/node001.nix. Keep the endpoint lists in sync. +storage: + type: sqlite + path: /data/data.db + +ui: + title: Status (homelab) + header: s2 · homelab + buttons: + - name: s1 (Hetzner) + link: https://s1.rubenhensen.nl + +endpoints: + # Tunneled through Hetzner to the home cluster + - name: rss + group: rubenhensen.nl + url: https://rss.rubenhensen.nl + interval: 60s + # mod_auth_openidc answers 401 instead of redirecting to the login + # page unless the client asks for HTML + headers: + Accept: text/html + conditions: &http-ok + - "[STATUS] == 200" + - "[CERTIFICATE_EXPIRATION] > 168h" + - name: authentik + group: rubenhensen.nl + url: https://authentik.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: vault + group: rubenhensen.nl + url: https://vault.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: ynab + group: rubenhensen.nl + url: https://ynab.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: argocd + group: rubenhensen.nl + url: https://argocd.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: ha + group: rubenhensen.nl + url: https://ha.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: longhorn + group: rubenhensen.nl + url: https://longhorn.rubenhensen.nl + interval: 60s + # Behind basic auth, so 401 means it's up + conditions: + - "[STATUS] == 401" + - "[CERTIFICATE_EXPIRATION] > 168h" + - name: lingo + group: rubenhensen.nl + url: https://lingo.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: blog + group: rubenhensen.nl + url: https://blog.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: serpbear + group: rubenhensen.nl + url: https://serpbear.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: git + group: rubenhensen.nl + url: https://git.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: vic + group: rubenhensen.nl + url: https://vic.rubenhensen.nl + interval: 60s + conditions: *http-ok + + # Stalwart on Hetzner + - name: webmail + group: mail + url: https://mail.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: smtp + group: mail + url: tcp://mail.rubenhensen.nl:25 + interval: 60s + conditions: + - "[CONNECTED] == true" + - name: imaps + group: mail + url: tls://mail.rubenhensen.nl:993 + interval: 60s + conditions: + - "[CONNECTED] == true" + - "[CERTIFICATE_EXPIRATION] > 168h" + + # Served straight from the home IP + - name: scoutingmbg.nl + group: Scouting MBG + url: https://scoutingmbg.nl + interval: 60s + conditions: *http-ok + - name: foto.scoutingmbg.nl + group: Scouting MBG + url: https://foto.scoutingmbg.nl/_app + interval: 60s + conditions: *http-ok + + # The other watcher + - name: s1 (Hetzner) + group: watchers + url: https://s1.rubenhensen.nl/health + interval: 60s + conditions: + - "[STATUS] == 200" + - "[BODY].status == UP" diff --git a/gatus/data-persistentvolumeclaim.yaml b/gatus/data-persistentvolumeclaim.yaml new file mode 100644 index 0000000..cc7cf76 --- /dev/null +++ b/gatus/data-persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gatus-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/gatus/gatus-deployment.yaml b/gatus/gatus-deployment.yaml new file mode 100644 index 0000000..af74157 --- /dev/null +++ b/gatus/gatus-deployment.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gatus +spec: + replicas: 1 + selector: + matchLabels: + app: gatus + strategy: + type: Recreate + template: + metadata: + labels: + app: gatus + spec: + containers: + - name: gatus + image: twinproduction/gatus:v5.37.0 + env: + - name: GATUS_CONFIG_PATH + value: /config/config.yaml + - name: TZ + value: Europe/Amsterdam + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + livenessProbe: + httpGet: + path: /health + port: 8080 + resources: + requests: + memory: "64Mi" + cpu: "20m" + limits: + memory: "256Mi" + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: gatus-config + - name: data + persistentVolumeClaim: + claimName: gatus-data diff --git a/gatus/gatus-service.yaml b/gatus/gatus-service.yaml new file mode 100644 index 0000000..84a19f6 --- /dev/null +++ b/gatus/gatus-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: gatus +spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + app: gatus diff --git a/gatus/ingress.yaml b/gatus/ingress.yaml new file mode 100644 index 0000000..a15e5d7 --- /dev/null +++ b/gatus/ingress.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: gatus-ingress + annotations: + cert-manager.io/cluster-issuer: prod-cluster-issuer + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/ssl-passthrough: "false" +spec: + ingressClassName: nginx + rules: + - host: s2.rubenhensen.nl + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gatus + port: + number: 8080 + tls: + - secretName: letsencrypt-prod + hosts: + - s2.rubenhensen.nl diff --git a/gatus/kustomization.yaml b/gatus/kustomization.yaml new file mode 100644 index 0000000..b136632 --- /dev/null +++ b/gatus/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: gatus + +resources: + - data-persistentvolumeclaim.yaml + - gatus-deployment.yaml + - gatus-service.yaml + - ingress.yaml + +configMapGenerator: + - name: gatus-config + files: + - config.yaml diff --git a/renovate.json b/renovate.json index 58977ed..ee51e2f 100644 --- a/renovate.json +++ b/renovate.json @@ -18,7 +18,8 @@ "/apps/templates/.+\\.yaml$/", "/serpbear/.+\\.yaml$/", "/bolletjestrui/.+\\.yaml$/", - "/gitea/.+\\.yaml$/" + "/gitea/.+\\.yaml$/", + "/gatus/.+\\.yaml$/" ] }, "customManagers": [