gatus: alert through self-hosted ntfy

Add ntfy at ntfy2.rubenhensen.nl on the home cluster, pointed straight at
the home IP like s2. s2 now alerts on mail and on s1 through it. The
home services are left to s1, which sees them from outside.

ntfy denies everything except the provisioned user. The user, its
password hash and the Gatus token come from Vault kv/ntfy.

apps/templates/ntfy-folder.yaml

@@ -0,0 +1,21 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ntfy-folder
+  namespace: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/rubenhensen/k8scd.git
+    targetRevision: HEAD
+    path: ntfy
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ntfy
+  syncPolicy:
+    syncOptions:
+    - CreateNamespace=true
+    automated:
+      selfHeal: true

dns/domains/rubenhensen.nl.yaml

@@ -73,6 +73,15 @@ records:
     expire: 300
     type: A
     content: "62.41.86.27"
+  # ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
+  - name: "ntfy1"
+    expire: 300
+    type: A
+    content: "46.224.26.65"
+  - name: "ntfy2"
+    expire: 300
+    type: A
+    content: "62.41.86.27"
   - name: "@"
     expire: 300
     type: MX

gatus/README.md

@@ -15,6 +15,32 @@ The endpoint list exists twice, so a new service has to be added in both
 places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up
 there automatically. s2 needs them added by hand.
 
+## Alerting
+
+Each Gatus sends alerts to an ntfy instance on its own node:
+`ntfy1.rubenhensen.nl` (Hetzner, node001.nix) and `ntfy2.rubenhensen.nl`
+(home, `ntfy/` in this repo). Subscribe the phone app to topic `alerts` on
+both servers, with the same login on each.
+
+Only the watcher that sees a service from outside alerts on it:
+
+- s1 alerts on everything that runs at home (all tunneled apps, Scouting MBG)
+  and on s2.
+- s2 alerts on mail and on s1.
+
+An outage on either node still reaches you, through the other node's ntfy,
+and you get one notification per outage instead of two. An alert fires after
+3 failed checks in a row (about 3 minutes), and you get another notification
+when the service recovers.
+
+Both ntfy servers deny everything except the provisioned user. The
+credentials are the same on both:
+
+- Home: Vault `kv/ntfy` with `auth_users`, `auth_tokens` and `gatus_token`,
+  pulled in by External Secrets.
+- Hetzner: the nix-infra secrets `ntfy-auth-users`, `ntfy-auth-tokens` and
+  `ntfy-gatus-token`.
+
 ## Reading the two pages
 
 - s1 sees the homelab from outside, so trust it for homelab services. s2 sees

gatus/config.yaml

@@ -12,6 +12,21 @@ ui:
     - name: s1 (Hetzner)
       link: https://s1.rubenhensen.nl
 
+# NTFY_TOKEN comes from the gatus-ntfy secret (Vault kv/ntfy)
+alerting:
+  ntfy:
+    url: http://ntfy.ntfy.svc.cluster.local
+    topic: alerts
+    token: ${NTFY_TOKEN}
+    priority: 4
+    click: https://s2.rubenhensen.nl
+    default-alert:
+      failure-threshold: 3
+      success-threshold: 2
+      send-on-resolved: true
+
+# s2 is the outside view of Hetzner, so it alerts on mail and on s1. The
+# home services are left to s1.
 endpoints:
   # Tunneled through Hetzner to the home cluster
   - name: rss
@@ -90,12 +105,15 @@ endpoints:
     url: https://mail.rubenhensen.nl
     interval: 60s
     conditions: *http-ok
+    alerts: &ntfy
+      - type: ntfy
   - name: smtp
     group: mail
     url: tcp://mail.rubenhensen.nl:25
     interval: 60s
     conditions:
       - "[CONNECTED] == true"
+    alerts: *ntfy
   - name: imaps
     group: mail
     url: tls://mail.rubenhensen.nl:993
@@ -103,6 +121,7 @@ endpoints:
     conditions:
       - "[CONNECTED] == true"
       - "[CERTIFICATE_EXPIRATION] > 168h"
+    alerts: *ntfy
 
   # Served straight from the home IP
   - name: scoutingmbg.nl
@@ -124,3 +143,4 @@ endpoints:
     conditions:
       - "[STATUS] == 200"
       - "[BODY].status == UP"
+    alerts: *ntfy

gatus/external-secret.yaml

@@ -0,0 +1,17 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: gatus-ntfy
+spec:
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  refreshInterval: 15m
+  target:
+    name: gatus-ntfy
+    creationPolicy: Owner
+  data:
+    - secretKey: NTFY_TOKEN
+      remoteRef:
+        key: kv/ntfy
+        property: gatus_token

gatus/gatus-deployment.yaml

@@ -22,6 +22,9 @@ spec:
               value: /config/config.yaml
             - name: TZ
               value: Europe/Amsterdam
+          envFrom:
+            - secretRef:
+                name: gatus-ntfy
           ports:
             - containerPort: 8080
               protocol: TCP

gatus/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 namespace: gatus
 
 resources:
+  - external-secret.yaml
   - data-persistentvolumeclaim.yaml
   - gatus-deployment.yaml
   - gatus-service.yaml

ntfy/data-persistentvolumeclaim.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ntfy-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi

ntfy/external-secret.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: ntfy-auth
+spec:
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  refreshInterval: 15m
+  target:
+    name: ntfy-auth
+    creationPolicy: Owner
+  data:
+    - secretKey: NTFY_AUTH_USERS
+      remoteRef:
+        key: kv/ntfy
+        property: auth_users
+    - secretKey: NTFY_AUTH_TOKENS
+      remoteRef:
+        key: kv/ntfy
+        property: auth_tokens

ntfy/ingress.yaml

@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ntfy-ingress
+  annotations:
+    cert-manager.io/cluster-issuer: prod-cluster-issuer
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "false"
+    # Keep subscriptions open, as in the ntfy proxy docs
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "180"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "180"
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: ntfy2.rubenhensen.nl
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: ntfy
+            port:
+              number: 80
+  tls:
+    - secretName: letsencrypt-prod
+      hosts:
+        - ntfy2.rubenhensen.nl

ntfy/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ntfy
+
+resources:
+  - external-secret.yaml
+  - data-persistentvolumeclaim.yaml
+  - ntfy-deployment.yaml
+  - ntfy-service.yaml
+  - ingress.yaml

ntfy/ntfy-deployment.yaml

@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ntfy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ntfy
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: ntfy
+    spec:
+      containers:
+        - name: ntfy
+          image: binwiederhier/ntfy:v2.28.0
+          args: ["serve"]
+          env:
+            - name: NTFY_BASE_URL
+              value: https://ntfy2.rubenhensen.nl
+            - name: NTFY_LISTEN_HTTP
+              value: ":80"
+            - name: NTFY_BEHIND_PROXY
+              value: "true"
+            - name: NTFY_AUTH_FILE
+              value: /var/lib/ntfy/user.db
+            - name: NTFY_AUTH_DEFAULT_ACCESS
+              value: deny-all
+            - name: NTFY_CACHE_FILE
+              value: /var/lib/ntfy/cache.db
+            # Needed for instant delivery on iOS. ntfy.sh only gets the
+            # message ID, never the content.
+            - name: NTFY_UPSTREAM_BASE_URL
+              value: https://ntfy.sh
+            - name: TZ
+              value: Europe/Amsterdam
+          envFrom:
+            - secretRef:
+                name: ntfy-auth
+          ports:
+            - containerPort: 80
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /v1/health
+              port: 80
+          livenessProbe:
+            httpGet:
+              path: /v1/health
+              port: 80
+          resources:
+            requests:
+              memory: "32Mi"
+              cpu: "10m"
+            limits:
+              memory: "256Mi"
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/ntfy
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: ntfy-data

ntfy/ntfy-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ntfy
+spec:
+  ports:
+    - name: http
+      port: 80
+      targetPort: 80
+  selector:
+    app: ntfy

renovate.json

@@ -19,7 +19,8 @@
       "/serpbear/.+\\.yaml$/",
       "/bolletjestrui/.+\\.yaml$/",
       "/gitea/.+\\.yaml$/",
-      "/gatus/.+\\.yaml$/"
+      "/gatus/.+\\.yaml$/",
+      "/ntfy/.+\\.yaml$/"
     ]
   },
   "customManagers": [