From e23779044fdeca5cbf41d4ba8946e6d89e644dd8 Mon Sep 17 00:00:00 2001 From: Ruben Hensen Date: Tue, 20 Oct 2026 15:34:42 +0200 Subject: [PATCH] gatus: alert through self-hosted ntfy Add ntfy at ntfy2.rubenhensen.nl on the home cluster, pointed straight at the home IP like s2. s2 now alerts on mail and on s1 through it. The home services are left to s1, which sees them from outside. ntfy denies everything except the provisioned user. The user, its password hash and the Gatus token come from Vault kv/ntfy. --- apps/templates/ntfy-folder.yaml | 21 +++++++++ dns/domains/rubenhensen.nl.yaml | 9 ++++ gatus/README.md | 26 +++++++++++ gatus/config.yaml | 20 +++++++++ gatus/external-secret.yaml | 17 +++++++ gatus/gatus-deployment.yaml | 3 ++ gatus/kustomization.yaml | 1 + ntfy/data-persistentvolumeclaim.yaml | 10 +++++ ntfy/external-secret.yaml | 21 +++++++++ ntfy/ingress.yaml | 28 ++++++++++++ ntfy/kustomization.yaml | 11 +++++ ntfy/ntfy-deployment.yaml | 66 ++++++++++++++++++++++++++++ ntfy/ntfy-service.yaml | 11 +++++ renovate.json | 3 +- 14 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 apps/templates/ntfy-folder.yaml create mode 100644 gatus/external-secret.yaml create mode 100644 ntfy/data-persistentvolumeclaim.yaml create mode 100644 ntfy/external-secret.yaml create mode 100644 ntfy/ingress.yaml create mode 100644 ntfy/kustomization.yaml create mode 100644 ntfy/ntfy-deployment.yaml create mode 100644 ntfy/ntfy-service.yaml diff --git a/apps/templates/ntfy-folder.yaml b/apps/templates/ntfy-folder.yaml new file mode 100644 index 0000000..60afb1a --- /dev/null +++ b/apps/templates/ntfy-folder.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ntfy-folder + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/rubenhensen/k8scd.git + targetRevision: HEAD + path: ntfy + destination: + server: https://kubernetes.default.svc + namespace: ntfy + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true diff --git a/dns/domains/rubenhensen.nl.yaml b/dns/domains/rubenhensen.nl.yaml index 59809ab..244cdb3 100644 --- a/dns/domains/rubenhensen.nl.yaml +++ b/dns/domains/rubenhensen.nl.yaml @@ -73,6 +73,15 @@ records: expire: 300 type: A content: "62.41.86.27" + # ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2. + - name: "ntfy1" + expire: 300 + type: A + content: "46.224.26.65" + - name: "ntfy2" + expire: 300 + type: A + content: "62.41.86.27" - name: "@" expire: 300 type: MX diff --git a/gatus/README.md b/gatus/README.md index 62dc78c..0a0ee2d 100644 --- a/gatus/README.md +++ b/gatus/README.md @@ -15,6 +15,32 @@ The endpoint list exists twice, so a new service has to be added in both places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up there automatically. s2 needs them added by hand. +## Alerting + +Each Gatus sends alerts to an ntfy instance on its own node: +`ntfy1.rubenhensen.nl` (Hetzner, node001.nix) and `ntfy2.rubenhensen.nl` +(home, `ntfy/` in this repo). Subscribe the phone app to topic `alerts` on +both servers, with the same login on each. + +Only the watcher that sees a service from outside alerts on it: + +- s1 alerts on everything that runs at home (all tunneled apps, Scouting MBG) + and on s2. +- s2 alerts on mail and on s1. + +An outage on either node still reaches you, through the other node's ntfy, +and you get one notification per outage instead of two. An alert fires after +3 failed checks in a row (about 3 minutes), and you get another notification +when the service recovers. + +Both ntfy servers deny everything except the provisioned user. The +credentials are the same on both: + +- Home: Vault `kv/ntfy` with `auth_users`, `auth_tokens` and `gatus_token`, + pulled in by External Secrets. +- Hetzner: the nix-infra secrets `ntfy-auth-users`, `ntfy-auth-tokens` and + `ntfy-gatus-token`. + ## Reading the two pages - s1 sees the homelab from outside, so trust it for homelab services. s2 sees diff --git a/gatus/config.yaml b/gatus/config.yaml index d0712bc..24b3078 100644 --- a/gatus/config.yaml +++ b/gatus/config.yaml @@ -12,6 +12,21 @@ ui: - name: s1 (Hetzner) link: https://s1.rubenhensen.nl +# NTFY_TOKEN comes from the gatus-ntfy secret (Vault kv/ntfy) +alerting: + ntfy: + url: http://ntfy.ntfy.svc.cluster.local + topic: alerts + token: ${NTFY_TOKEN} + priority: 4 + click: https://s2.rubenhensen.nl + default-alert: + failure-threshold: 3 + success-threshold: 2 + send-on-resolved: true + +# s2 is the outside view of Hetzner, so it alerts on mail and on s1. The +# home services are left to s1. endpoints: # Tunneled through Hetzner to the home cluster - name: rss @@ -90,12 +105,15 @@ endpoints: url: https://mail.rubenhensen.nl interval: 60s conditions: *http-ok + alerts: &ntfy + - type: ntfy - name: smtp group: mail url: tcp://mail.rubenhensen.nl:25 interval: 60s conditions: - "[CONNECTED] == true" + alerts: *ntfy - name: imaps group: mail url: tls://mail.rubenhensen.nl:993 @@ -103,6 +121,7 @@ endpoints: conditions: - "[CONNECTED] == true" - "[CERTIFICATE_EXPIRATION] > 168h" + alerts: *ntfy # Served straight from the home IP - name: scoutingmbg.nl @@ -124,3 +143,4 @@ endpoints: conditions: - "[STATUS] == 200" - "[BODY].status == UP" + alerts: *ntfy diff --git a/gatus/external-secret.yaml b/gatus/external-secret.yaml new file mode 100644 index 0000000..a184dce --- /dev/null +++ b/gatus/external-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: gatus-ntfy +spec: + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + refreshInterval: 15m + target: + name: gatus-ntfy + creationPolicy: Owner + data: + - secretKey: NTFY_TOKEN + remoteRef: + key: kv/ntfy + property: gatus_token diff --git a/gatus/gatus-deployment.yaml b/gatus/gatus-deployment.yaml index af74157..6555bb3 100644 --- a/gatus/gatus-deployment.yaml +++ b/gatus/gatus-deployment.yaml @@ -22,6 +22,9 @@ spec: value: /config/config.yaml - name: TZ value: Europe/Amsterdam + envFrom: + - secretRef: + name: gatus-ntfy ports: - containerPort: 8080 protocol: TCP diff --git a/gatus/kustomization.yaml b/gatus/kustomization.yaml index b136632..6a58f9a 100644 --- a/gatus/kustomization.yaml +++ b/gatus/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization namespace: gatus resources: + - external-secret.yaml - data-persistentvolumeclaim.yaml - gatus-deployment.yaml - gatus-service.yaml diff --git a/ntfy/data-persistentvolumeclaim.yaml b/ntfy/data-persistentvolumeclaim.yaml new file mode 100644 index 0000000..e6515a0 --- /dev/null +++ b/ntfy/data-persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ntfy-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/ntfy/external-secret.yaml b/ntfy/external-secret.yaml new file mode 100644 index 0000000..2b441f7 --- /dev/null +++ b/ntfy/external-secret.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: ntfy-auth +spec: + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + refreshInterval: 15m + target: + name: ntfy-auth + creationPolicy: Owner + data: + - secretKey: NTFY_AUTH_USERS + remoteRef: + key: kv/ntfy + property: auth_users + - secretKey: NTFY_AUTH_TOKENS + remoteRef: + key: kv/ntfy + property: auth_tokens diff --git a/ntfy/ingress.yaml b/ntfy/ingress.yaml new file mode 100644 index 0000000..4ab0769 --- /dev/null +++ b/ntfy/ingress.yaml @@ -0,0 +1,28 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ntfy-ingress + annotations: + cert-manager.io/cluster-issuer: prod-cluster-issuer + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/ssl-passthrough: "false" + # Keep subscriptions open, as in the ntfy proxy docs + nginx.ingress.kubernetes.io/proxy-read-timeout: "180" + nginx.ingress.kubernetes.io/proxy-send-timeout: "180" +spec: + ingressClassName: nginx + rules: + - host: ntfy2.rubenhensen.nl + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ntfy + port: + number: 80 + tls: + - secretName: letsencrypt-prod + hosts: + - ntfy2.rubenhensen.nl diff --git a/ntfy/kustomization.yaml b/ntfy/kustomization.yaml new file mode 100644 index 0000000..4c25539 --- /dev/null +++ b/ntfy/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ntfy + +resources: + - external-secret.yaml + - data-persistentvolumeclaim.yaml + - ntfy-deployment.yaml + - ntfy-service.yaml + - ingress.yaml diff --git a/ntfy/ntfy-deployment.yaml b/ntfy/ntfy-deployment.yaml new file mode 100644 index 0000000..b055155 --- /dev/null +++ b/ntfy/ntfy-deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ntfy +spec: + replicas: 1 + selector: + matchLabels: + app: ntfy + strategy: + type: Recreate + template: + metadata: + labels: + app: ntfy + spec: + containers: + - name: ntfy + image: binwiederhier/ntfy:v2.28.0 + args: ["serve"] + env: + - name: NTFY_BASE_URL + value: https://ntfy2.rubenhensen.nl + - name: NTFY_LISTEN_HTTP + value: ":80" + - name: NTFY_BEHIND_PROXY + value: "true" + - name: NTFY_AUTH_FILE + value: /var/lib/ntfy/user.db + - name: NTFY_AUTH_DEFAULT_ACCESS + value: deny-all + - name: NTFY_CACHE_FILE + value: /var/lib/ntfy/cache.db + # Needed for instant delivery on iOS. ntfy.sh only gets the + # message ID, never the content. + - name: NTFY_UPSTREAM_BASE_URL + value: https://ntfy.sh + - name: TZ + value: Europe/Amsterdam + envFrom: + - secretRef: + name: ntfy-auth + ports: + - containerPort: 80 + protocol: TCP + readinessProbe: + httpGet: + path: /v1/health + port: 80 + livenessProbe: + httpGet: + path: /v1/health + port: 80 + resources: + requests: + memory: "32Mi" + cpu: "10m" + limits: + memory: "256Mi" + volumeMounts: + - name: data + mountPath: /var/lib/ntfy + volumes: + - name: data + persistentVolumeClaim: + claimName: ntfy-data diff --git a/ntfy/ntfy-service.yaml b/ntfy/ntfy-service.yaml new file mode 100644 index 0000000..09672b4 --- /dev/null +++ b/ntfy/ntfy-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: ntfy +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app: ntfy diff --git a/renovate.json b/renovate.json index ee51e2f..b2ca4ec 100644 --- a/renovate.json +++ b/renovate.json @@ -19,7 +19,8 @@ "/serpbear/.+\\.yaml$/", "/bolletjestrui/.+\\.yaml$/", "/gitea/.+\\.yaml$/", - "/gatus/.+\\.yaml$/" + "/gatus/.+\\.yaml$/", + "/ntfy/.+\\.yaml$/" ] }, "customManagers": [