diff --git a/apps/templates/ntfy-folder.yaml b/apps/templates/ntfy-folder.yaml new file mode 100644 index 0000000..60afb1a --- /dev/null +++ b/apps/templates/ntfy-folder.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ntfy-folder + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/rubenhensen/k8scd.git + targetRevision: HEAD + path: ntfy + destination: + server: https://kubernetes.default.svc + namespace: ntfy + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true diff --git a/dns/domains/rubenhensen.nl.yaml b/dns/domains/rubenhensen.nl.yaml index 59809ab..244cdb3 100644 --- a/dns/domains/rubenhensen.nl.yaml +++ b/dns/domains/rubenhensen.nl.yaml @@ -73,6 +73,15 @@ records: expire: 300 type: A content: "62.41.86.27" + # ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2. + - name: "ntfy1" + expire: 300 + type: A + content: "46.224.26.65" + - name: "ntfy2" + expire: 300 + type: A + content: "62.41.86.27" - name: "@" expire: 300 type: MX diff --git a/gatus/README.md b/gatus/README.md index 62dc78c..0a0ee2d 100644 --- a/gatus/README.md +++ b/gatus/README.md @@ -15,6 +15,32 @@ The endpoint list exists twice, so a new service has to be added in both places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up there automatically. s2 needs them added by hand. +## Alerting + +Each Gatus sends alerts to an ntfy instance on its own node: +`ntfy1.rubenhensen.nl` (Hetzner, node001.nix) and `ntfy2.rubenhensen.nl` +(home, `ntfy/` in this repo). Subscribe the phone app to topic `alerts` on +both servers, with the same login on each. + +Only the watcher that sees a service from outside alerts on it: + +- s1 alerts on everything that runs at home (all tunneled apps, Scouting MBG) + and on s2. +- s2 alerts on mail and on s1. + +An outage on either node still reaches you, through the other node's ntfy, +and you get one notification per outage instead of two. An alert fires after +3 failed checks in a row (about 3 minutes), and you get another notification +when the service recovers. + +Both ntfy servers deny everything except the provisioned user. The +credentials are the same on both: + +- Home: Vault `kv/ntfy` with `auth_users`, `auth_tokens` and `gatus_token`, + pulled in by External Secrets. +- Hetzner: the nix-infra secrets `ntfy-auth-users`, `ntfy-auth-tokens` and + `ntfy-gatus-token`. + ## Reading the two pages - s1 sees the homelab from outside, so trust it for homelab services. s2 sees diff --git a/gatus/config.yaml b/gatus/config.yaml index d0712bc..24b3078 100644 --- a/gatus/config.yaml +++ b/gatus/config.yaml @@ -12,6 +12,21 @@ ui: - name: s1 (Hetzner) link: https://s1.rubenhensen.nl +# NTFY_TOKEN comes from the gatus-ntfy secret (Vault kv/ntfy) +alerting: + ntfy: + url: http://ntfy.ntfy.svc.cluster.local + topic: alerts + token: ${NTFY_TOKEN} + priority: 4 + click: https://s2.rubenhensen.nl + default-alert: + failure-threshold: 3 + success-threshold: 2 + send-on-resolved: true + +# s2 is the outside view of Hetzner, so it alerts on mail and on s1. The +# home services are left to s1. endpoints: # Tunneled through Hetzner to the home cluster - name: rss @@ -90,12 +105,15 @@ endpoints: url: https://mail.rubenhensen.nl interval: 60s conditions: *http-ok + alerts: &ntfy + - type: ntfy - name: smtp group: mail url: tcp://mail.rubenhensen.nl:25 interval: 60s conditions: - "[CONNECTED] == true" + alerts: *ntfy - name: imaps group: mail url: tls://mail.rubenhensen.nl:993 @@ -103,6 +121,7 @@ endpoints: conditions: - "[CONNECTED] == true" - "[CERTIFICATE_EXPIRATION] > 168h" + alerts: *ntfy # Served straight from the home IP - name: scoutingmbg.nl @@ -124,3 +143,4 @@ endpoints: conditions: - "[STATUS] == 200" - "[BODY].status == UP" + alerts: *ntfy diff --git a/gatus/external-secret.yaml b/gatus/external-secret.yaml new file mode 100644 index 0000000..a184dce --- /dev/null +++ b/gatus/external-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: gatus-ntfy +spec: + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + refreshInterval: 15m + target: + name: gatus-ntfy + creationPolicy: Owner + data: + - secretKey: NTFY_TOKEN + remoteRef: + key: kv/ntfy + property: gatus_token diff --git a/gatus/gatus-deployment.yaml b/gatus/gatus-deployment.yaml index af74157..6555bb3 100644 --- a/gatus/gatus-deployment.yaml +++ b/gatus/gatus-deployment.yaml @@ -22,6 +22,9 @@ spec: value: /config/config.yaml - name: TZ value: Europe/Amsterdam + envFrom: + - secretRef: + name: gatus-ntfy ports: - containerPort: 8080 protocol: TCP diff --git a/gatus/kustomization.yaml b/gatus/kustomization.yaml index b136632..6a58f9a 100644 --- a/gatus/kustomization.yaml +++ b/gatus/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization namespace: gatus resources: + - external-secret.yaml - data-persistentvolumeclaim.yaml - gatus-deployment.yaml - gatus-service.yaml diff --git a/ntfy/data-persistentvolumeclaim.yaml b/ntfy/data-persistentvolumeclaim.yaml new file mode 100644 index 0000000..e6515a0 --- /dev/null +++ b/ntfy/data-persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ntfy-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/ntfy/external-secret.yaml b/ntfy/external-secret.yaml new file mode 100644 index 0000000..2b441f7 --- /dev/null +++ b/ntfy/external-secret.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: ntfy-auth +spec: + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + refreshInterval: 15m + target: + name: ntfy-auth + creationPolicy: Owner + data: + - secretKey: NTFY_AUTH_USERS + remoteRef: + key: kv/ntfy + property: auth_users + - secretKey: NTFY_AUTH_TOKENS + remoteRef: + key: kv/ntfy + property: auth_tokens diff --git a/ntfy/ingress.yaml b/ntfy/ingress.yaml new file mode 100644 index 0000000..4ab0769 --- /dev/null +++ b/ntfy/ingress.yaml @@ -0,0 +1,28 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ntfy-ingress + annotations: + cert-manager.io/cluster-issuer: prod-cluster-issuer + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/ssl-passthrough: "false" + # Keep subscriptions open, as in the ntfy proxy docs + nginx.ingress.kubernetes.io/proxy-read-timeout: "180" + nginx.ingress.kubernetes.io/proxy-send-timeout: "180" +spec: + ingressClassName: nginx + rules: + - host: ntfy2.rubenhensen.nl + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ntfy + port: + number: 80 + tls: + - secretName: letsencrypt-prod + hosts: + - ntfy2.rubenhensen.nl diff --git a/ntfy/kustomization.yaml b/ntfy/kustomization.yaml new file mode 100644 index 0000000..4c25539 --- /dev/null +++ b/ntfy/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ntfy + +resources: + - external-secret.yaml + - data-persistentvolumeclaim.yaml + - ntfy-deployment.yaml + - ntfy-service.yaml + - ingress.yaml diff --git a/ntfy/ntfy-deployment.yaml b/ntfy/ntfy-deployment.yaml new file mode 100644 index 0000000..b055155 --- /dev/null +++ b/ntfy/ntfy-deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ntfy +spec: + replicas: 1 + selector: + matchLabels: + app: ntfy + strategy: + type: Recreate + template: + metadata: + labels: + app: ntfy + spec: + containers: + - name: ntfy + image: binwiederhier/ntfy:v2.28.0 + args: ["serve"] + env: + - name: NTFY_BASE_URL + value: https://ntfy2.rubenhensen.nl + - name: NTFY_LISTEN_HTTP + value: ":80" + - name: NTFY_BEHIND_PROXY + value: "true" + - name: NTFY_AUTH_FILE + value: /var/lib/ntfy/user.db + - name: NTFY_AUTH_DEFAULT_ACCESS + value: deny-all + - name: NTFY_CACHE_FILE + value: /var/lib/ntfy/cache.db + # Needed for instant delivery on iOS. ntfy.sh only gets the + # message ID, never the content. + - name: NTFY_UPSTREAM_BASE_URL + value: https://ntfy.sh + - name: TZ + value: Europe/Amsterdam + envFrom: + - secretRef: + name: ntfy-auth + ports: + - containerPort: 80 + protocol: TCP + readinessProbe: + httpGet: + path: /v1/health + port: 80 + livenessProbe: + httpGet: + path: /v1/health + port: 80 + resources: + requests: + memory: "32Mi" + cpu: "10m" + limits: + memory: "256Mi" + volumeMounts: + - name: data + mountPath: /var/lib/ntfy + volumes: + - name: data + persistentVolumeClaim: + claimName: ntfy-data diff --git a/ntfy/ntfy-service.yaml b/ntfy/ntfy-service.yaml new file mode 100644 index 0000000..09672b4 --- /dev/null +++ b/ntfy/ntfy-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: ntfy +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app: ntfy diff --git a/renovate.json b/renovate.json index ee51e2f..b2ca4ec 100644 --- a/renovate.json +++ b/renovate.json @@ -19,7 +19,8 @@ "/serpbear/.+\\.yaml$/", "/bolletjestrui/.+\\.yaml$/", "/gitea/.+\\.yaml$/", - "/gatus/.+\\.yaml$/" + "/gatus/.+\\.yaml$/", + "/ntfy/.+\\.yaml$/" ] }, "customManagers": [