diff --git a/apps/templates/gatus-folder.yaml b/apps/templates/gatus-folder.yaml new file mode 100644 index 0000000..be630ef --- /dev/null +++ b/apps/templates/gatus-folder.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: gatus-folder + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/rubenhensen/k8scd.git + targetRevision: HEAD + path: gatus + destination: + server: https://kubernetes.default.svc + namespace: gatus + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true diff --git a/dns/domains/rubenhensen.nl.yaml b/dns/domains/rubenhensen.nl.yaml index c134432..59809ab 100644 --- a/dns/domains/rubenhensen.nl.yaml +++ b/dns/domains/rubenhensen.nl.yaml @@ -63,6 +63,16 @@ records: expire: 300 type: A content: "46.224.26.65" + # Gatus status pages. s2 points straight at home, not through the + # Hetzner tunnel, so it stays reachable when Hetzner is down. + - name: "s1" + expire: 300 + type: A + content: "46.224.26.65" + - name: "s2" + expire: 300 + type: A + content: "62.41.86.27" - name: "@" expire: 300 type: MX diff --git a/gatus/README.md b/gatus/README.md new file mode 100644 index 0000000..62dc78c --- /dev/null +++ b/gatus/README.md @@ -0,0 +1,79 @@ +# Gatus + +Two Gatus instances check the same endpoints from two places: + +| Status page | Runs on | Config | +|---|---|---| +| https://s1.rubenhensen.nl | Hetzner (node001) | `nix-infra-machine/nodes/node001.nix` | +| https://s2.rubenhensen.nl | Home cluster | `gatus/config.yaml` (this folder) | + +Each one also checks the other's `/health`. The s2 DNS record points straight +at the home IP instead of going through the Hetzner tunnel, so s2 stays +reachable when Hetzner is down. + +The endpoint list exists twice, so a new service has to be added in both +places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up +there automatically. s2 needs them added by hand. + +## Reading the two pages + +- s1 sees the homelab from outside, so trust it for homelab services. s2 sees + Hetzner from outside, so trust it for mail and anything else on Hetzner. +- Every `*.rubenhensen.nl` app is tunneled Hetzner → home, so a Hetzner + outage shows all of them as down on both pages. That is accurate: nobody + can reach them then. +- If s1 and s2 disagree about an endpoint, the problem is the network path + between one watcher and that service, not the service itself. +- Neither page has history for the time its own node was down. + +## Later: one URL with self-hosted DNS failover + +This is the plan for replacing s1/s2 with a single `status.rubenhensen.nl` +that always points at a watcher that's up, with no third party involved. + +Hand `status.rubenhensen.nl` over to two nameservers we run ourselves, one per +node, both PowerDNS Authoritative. PowerDNS LUA records health-check +addresses and only return the healthy ones: + +``` +@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.27'}})" +``` + +The double braces make two groups. PowerDNS returns the first group that has +a healthy address, so Hetzner is primary and home is the fallback. Each +nameserver runs the check from its own location. + +| Situation | Hetzner NS | Home NS | Visitors end up on | +|---|---|---|---| +| Both up | answers Hetzner | answers Hetzner | Hetzner | +| Hetzner down | dead | Hetzner check fails, answers home | home | +| Home down / home internet down | answers Hetzner | dead | Hetzner | +| Hetzner up, its Gatus broken | own check fails, answers home | answers home | home | + +A node that goes down takes its nameserver down with it, so DNS resolvers +switch to the other nameserver. The two nodes never have to agree with each +other: the answer depends on what the internet can reach. The 30s TTL caps +how long clients keep a stale answer. + +Steps: + +1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add + `status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`, + plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.27). + First check that TransIP accepts NS records for a subdomain. +2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and + the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp. +3. Home: run PowerDNS in the cluster with the same zone file, on 53/udp and + 53/tcp of the home IP (router port forward, and check how the cluster + exposes UDP). +4. Both Gatus instances also serve `status.rubenhensen.nl`. On Hetzner, add the + name to the SNI map, the same way as s1. +5. TLS: Hetzner gets its cert through HTTP-01 as usual. On Hetzner's :80 vhost + for `status`, serve local ACME challenges and proxy the rest to the home + cluster (`try_files $uri @home`). That way cert-manager's HTTP-01 also + works while DNS points at Hetzner. When Hetzner is down, DNS points home + and cert-manager reaches the home cluster directly. +6. Requirements: a static home IP (`homeIP` in node001.nix already assumes + one), and port 53 open on both nodes. + +After that, s1/s2 can be removed, or kept as direct links to each instance. diff --git a/gatus/config.yaml b/gatus/config.yaml new file mode 100644 index 0000000..d0712bc --- /dev/null +++ b/gatus/config.yaml @@ -0,0 +1,126 @@ +# s2.rubenhensen.nl: Gatus on the home cluster. +# Its twin s1.rubenhensen.nl runs on the Hetzner box, see +# nix-infra-machine/nodes/node001.nix. Keep the endpoint lists in sync. +storage: + type: sqlite + path: /data/data.db + +ui: + title: Status (homelab) + header: s2 · homelab + buttons: + - name: s1 (Hetzner) + link: https://s1.rubenhensen.nl + +endpoints: + # Tunneled through Hetzner to the home cluster + - name: rss + group: rubenhensen.nl + url: https://rss.rubenhensen.nl + interval: 60s + # mod_auth_openidc answers 401 instead of redirecting to the login + # page unless the client asks for HTML + headers: + Accept: text/html + conditions: &http-ok + - "[STATUS] == 200" + - "[CERTIFICATE_EXPIRATION] > 168h" + - name: authentik + group: rubenhensen.nl + url: https://authentik.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: vault + group: rubenhensen.nl + url: https://vault.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: ynab + group: rubenhensen.nl + url: https://ynab.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: argocd + group: rubenhensen.nl + url: https://argocd.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: ha + group: rubenhensen.nl + url: https://ha.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: longhorn + group: rubenhensen.nl + url: https://longhorn.rubenhensen.nl + interval: 60s + # Behind basic auth, so 401 means it's up + conditions: + - "[STATUS] == 401" + - "[CERTIFICATE_EXPIRATION] > 168h" + - name: lingo + group: rubenhensen.nl + url: https://lingo.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: blog + group: rubenhensen.nl + url: https://blog.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: serpbear + group: rubenhensen.nl + url: https://serpbear.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: git + group: rubenhensen.nl + url: https://git.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: vic + group: rubenhensen.nl + url: https://vic.rubenhensen.nl + interval: 60s + conditions: *http-ok + + # Stalwart on Hetzner + - name: webmail + group: mail + url: https://mail.rubenhensen.nl + interval: 60s + conditions: *http-ok + - name: smtp + group: mail + url: tcp://mail.rubenhensen.nl:25 + interval: 60s + conditions: + - "[CONNECTED] == true" + - name: imaps + group: mail + url: tls://mail.rubenhensen.nl:993 + interval: 60s + conditions: + - "[CONNECTED] == true" + - "[CERTIFICATE_EXPIRATION] > 168h" + + # Served straight from the home IP + - name: scoutingmbg.nl + group: Scouting MBG + url: https://scoutingmbg.nl + interval: 60s + conditions: *http-ok + - name: foto.scoutingmbg.nl + group: Scouting MBG + url: https://foto.scoutingmbg.nl/_app + interval: 60s + conditions: *http-ok + + # The other watcher + - name: s1 (Hetzner) + group: watchers + url: https://s1.rubenhensen.nl/health + interval: 60s + conditions: + - "[STATUS] == 200" + - "[BODY].status == UP" diff --git a/gatus/data-persistentvolumeclaim.yaml b/gatus/data-persistentvolumeclaim.yaml new file mode 100644 index 0000000..cc7cf76 --- /dev/null +++ b/gatus/data-persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gatus-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/gatus/gatus-deployment.yaml b/gatus/gatus-deployment.yaml new file mode 100644 index 0000000..af74157 --- /dev/null +++ b/gatus/gatus-deployment.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gatus +spec: + replicas: 1 + selector: + matchLabels: + app: gatus + strategy: + type: Recreate + template: + metadata: + labels: + app: gatus + spec: + containers: + - name: gatus + image: twinproduction/gatus:v5.37.0 + env: + - name: GATUS_CONFIG_PATH + value: /config/config.yaml + - name: TZ + value: Europe/Amsterdam + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + livenessProbe: + httpGet: + path: /health + port: 8080 + resources: + requests: + memory: "64Mi" + cpu: "20m" + limits: + memory: "256Mi" + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: gatus-config + - name: data + persistentVolumeClaim: + claimName: gatus-data diff --git a/gatus/gatus-service.yaml b/gatus/gatus-service.yaml new file mode 100644 index 0000000..84a19f6 --- /dev/null +++ b/gatus/gatus-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: gatus +spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + app: gatus diff --git a/gatus/ingress.yaml b/gatus/ingress.yaml new file mode 100644 index 0000000..a15e5d7 --- /dev/null +++ b/gatus/ingress.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: gatus-ingress + annotations: + cert-manager.io/cluster-issuer: prod-cluster-issuer + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/ssl-passthrough: "false" +spec: + ingressClassName: nginx + rules: + - host: s2.rubenhensen.nl + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gatus + port: + number: 8080 + tls: + - secretName: letsencrypt-prod + hosts: + - s2.rubenhensen.nl diff --git a/gatus/kustomization.yaml b/gatus/kustomization.yaml new file mode 100644 index 0000000..b136632 --- /dev/null +++ b/gatus/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: gatus + +resources: + - data-persistentvolumeclaim.yaml + - gatus-deployment.yaml + - gatus-service.yaml + - ingress.yaml + +configMapGenerator: + - name: gatus-config + files: + - config.yaml diff --git a/renovate.json b/renovate.json index 58977ed..ee51e2f 100644 --- a/renovate.json +++ b/renovate.json @@ -18,7 +18,8 @@ "/apps/templates/.+\\.yaml$/", "/serpbear/.+\\.yaml$/", "/bolletjestrui/.+\\.yaml$/", - "/gitea/.+\\.yaml$/" + "/gitea/.+\\.yaml$/", + "/gatus/.+\\.yaml$/" ] }, "customManagers": [