mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 14:29:09 +01:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
054117ad6d | ||
|
|
ac61c346af | ||
|
|
e23779044f |
No files matched your search
@@ -0,0 +1,21 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: ntfy-folder
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: ntfy
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: ntfy
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -73,6 +73,15 @@ records:
|
||||
expire: 300
|
||||
type: A
|
||||
content: "62.41.86.27"
|
||||
# ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
|
||||
- name: "ntfy1"
|
||||
expire: 300
|
||||
type: A
|
||||
content: "46.224.26.65"
|
||||
- name: "ntfy2"
|
||||
expire: 300
|
||||
type: A
|
||||
content: "62.41.86.27"
|
||||
- name: "@"
|
||||
expire: 300
|
||||
type: MX
|
||||
|
||||
@@ -15,6 +15,32 @@ The endpoint list exists twice, so a new service has to be added in both
|
||||
places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up
|
||||
there automatically. s2 needs them added by hand.
|
||||
|
||||
## Alerting
|
||||
|
||||
Each Gatus sends alerts to an ntfy instance on its own node:
|
||||
`ntfy1.rubenhensen.nl` (Hetzner, node001.nix) and `ntfy2.rubenhensen.nl`
|
||||
(home, `ntfy/` in this repo). Subscribe the phone app to topic `alerts` on
|
||||
both servers, with the same login on each.
|
||||
|
||||
Only the watcher that sees a service from outside alerts on it:
|
||||
|
||||
- s1 alerts on everything that runs at home (all tunneled apps, Scouting MBG)
|
||||
and on s2.
|
||||
- s2 alerts on mail and on s1.
|
||||
|
||||
An outage on either node still reaches you, through the other node's ntfy,
|
||||
and you get one notification per outage instead of two. An alert fires after
|
||||
3 failed checks in a row (about 3 minutes), and you get another notification
|
||||
when the service recovers.
|
||||
|
||||
Both ntfy servers deny everything except the provisioned user. The
|
||||
credentials are the same on both:
|
||||
|
||||
- Home: Vault `kv/ntfy` with `auth_users`, `auth_tokens` and `gatus_token`,
|
||||
pulled in by External Secrets.
|
||||
- Hetzner: the nix-infra secrets `ntfy-auth-users`, `ntfy-auth-tokens` and
|
||||
`ntfy-gatus-token`.
|
||||
|
||||
## Reading the two pages
|
||||
|
||||
- s1 sees the homelab from outside, so trust it for homelab services. s2 sees
|
||||
|
||||
@@ -12,6 +12,21 @@ ui:
|
||||
- name: s1 (Hetzner)
|
||||
link: https://s1.rubenhensen.nl
|
||||
|
||||
# NTFY_TOKEN comes from the gatus-ntfy secret (Vault kv/ntfy)
|
||||
alerting:
|
||||
ntfy:
|
||||
url: http://ntfy.ntfy.svc.cluster.local
|
||||
topic: alerts
|
||||
token: ${NTFY_TOKEN}
|
||||
priority: 4
|
||||
click: https://s2.rubenhensen.nl
|
||||
default-alert:
|
||||
failure-threshold: 3
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
|
||||
# s2 is the outside view of Hetzner, so it alerts on mail and on s1. The
|
||||
# home services are left to s1.
|
||||
endpoints:
|
||||
# Tunneled through Hetzner to the home cluster
|
||||
- name: rss
|
||||
@@ -90,12 +105,15 @@ endpoints:
|
||||
url: https://mail.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
alerts: &ntfy
|
||||
- type: ntfy
|
||||
- name: smtp
|
||||
group: mail
|
||||
url: tcp://mail.rubenhensen.nl:25
|
||||
interval: 60s
|
||||
conditions:
|
||||
- "[CONNECTED] == true"
|
||||
alerts: *ntfy
|
||||
- name: imaps
|
||||
group: mail
|
||||
url: tls://mail.rubenhensen.nl:993
|
||||
@@ -103,6 +121,7 @@ endpoints:
|
||||
conditions:
|
||||
- "[CONNECTED] == true"
|
||||
- "[CERTIFICATE_EXPIRATION] > 168h"
|
||||
alerts: *ntfy
|
||||
|
||||
# Served straight from the home IP
|
||||
- name: scoutingmbg.nl
|
||||
@@ -124,3 +143,4 @@ endpoints:
|
||||
conditions:
|
||||
- "[STATUS] == 200"
|
||||
- "[BODY].status == UP"
|
||||
alerts: *ntfy
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: gatus-ntfy
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: gatus-ntfy
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: NTFY_TOKEN
|
||||
remoteRef:
|
||||
key: kv/ntfy
|
||||
property: gatus_token
|
||||
@@ -22,6 +22,9 @@ spec:
|
||||
value: /config/config.yaml
|
||||
- name: TZ
|
||||
value: Europe/Amsterdam
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: gatus-ntfy
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
|
||||
@@ -4,6 +4,7 @@ kind: Kustomization
|
||||
namespace: gatus
|
||||
|
||||
resources:
|
||||
- external-secret.yaml
|
||||
- data-persistentvolumeclaim.yaml
|
||||
- gatus-deployment.yaml
|
||||
- gatus-service.yaml
|
||||
|
||||
@@ -63,9 +63,14 @@ let
|
||||
"rss.rubenhensen.nl".headers.Accept = "text/html";
|
||||
};
|
||||
|
||||
# s1 is the outside view of home, so it alerts on what runs there and on
|
||||
# s2. Mail lives on this box, so s2 alerts on that.
|
||||
gatusAlert = { alerts = [ { type = "ntfy"; } ]; };
|
||||
|
||||
gatusEndpoints =
|
||||
map (h:
|
||||
gatusHttp "rubenhensen.nl" (lib.removeSuffix ".rubenhensen.nl" h) "https://${h}"
|
||||
// gatusAlert
|
||||
// (gatusOverrides.${h} or { })) tunneledHosts
|
||||
++ [
|
||||
(gatusHttp "mail" "webmail" "https://mail.rubenhensen.nl")
|
||||
@@ -83,16 +88,25 @@ let
|
||||
interval = "60s";
|
||||
conditions = [ "[CONNECTED] == true" "[CERTIFICATE_EXPIRATION] > 168h" ];
|
||||
}
|
||||
(gatusHttp "Scouting MBG" "scoutingmbg.nl" "https://scoutingmbg.nl")
|
||||
(gatusHttp "Scouting MBG" "foto.scoutingmbg.nl" "https://foto.scoutingmbg.nl/_app")
|
||||
{
|
||||
(gatusHttp "Scouting MBG" "scoutingmbg.nl" "https://scoutingmbg.nl" // gatusAlert)
|
||||
(gatusHttp "Scouting MBG" "foto.scoutingmbg.nl" "https://foto.scoutingmbg.nl/_app" // gatusAlert)
|
||||
({
|
||||
name = "s2 (homelab)";
|
||||
group = "watchers";
|
||||
url = "https://s2.rubenhensen.nl/health";
|
||||
interval = "60s";
|
||||
conditions = [ "[STATUS] == 200" "[BODY].status == UP" ];
|
||||
}
|
||||
} // gatusAlert)
|
||||
];
|
||||
|
||||
# Runs a service with secrets from LoadCredentialEncrypted exported as
|
||||
# env vars, for services that only take secrets through the environment.
|
||||
withCredentialEnv = name: vars: cmd: pkgs.writeShellScript "${name}-with-credentials" ''
|
||||
${lib.concatStrings (lib.mapAttrsToList (var: cred: ''
|
||||
export ${var}="$(cat "$CREDENTIALS_DIRECTORY/${cred}")"
|
||||
'') vars)}
|
||||
exec ${cmd}
|
||||
'';
|
||||
in
|
||||
{
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -141,11 +155,12 @@ in
|
||||
# SNI passthrough on 443:
|
||||
# - mail.rubenhensen.nl (and anything else) → local stalwart on 8443
|
||||
# - tunneled hosts → home cluster ingress on 443
|
||||
# - s1.rubenhensen.nl → local nginx on 8444 (Gatus)
|
||||
# - s1 / ntfy1.rubenhensen.nl → local nginx on 8444 (Gatus, ntfy)
|
||||
streamConfig = ''
|
||||
map $ssl_preread_server_name $tunnel_upstream {
|
||||
${sniMapEntries}
|
||||
s1.rubenhensen.nl 127.0.0.1:8444;
|
||||
ntfy1.rubenhensen.nl 127.0.0.1:8444;
|
||||
default 127.0.0.1:8443;
|
||||
}
|
||||
|
||||
@@ -206,9 +221,63 @@ ${sniMapEntries}
|
||||
forceSSL = true;
|
||||
locations."/".proxyPass = "http://127.0.0.1:8090";
|
||||
};
|
||||
# ntfy for Gatus alerts, same TLS setup as s1. Proxy settings follow
|
||||
# the ntfy docs so subscriptions can stay open.
|
||||
"ntfy1.rubenhensen.nl" = {
|
||||
listen = [
|
||||
{ addr = "0.0.0.0"; port = 80; }
|
||||
{ addr = "[::]"; port = 80; }
|
||||
{ addr = "127.0.0.1"; port = 8444; ssl = true; }
|
||||
];
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:2586";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = ''
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
proxy_connect_timeout 3m;
|
||||
proxy_send_timeout 3m;
|
||||
proxy_read_timeout 3m;
|
||||
client_max_body_size 0;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# ntfy (ntfy1.rubenhensen.nl)
|
||||
# ──────────────────────────────────────────────
|
||||
# Users and tokens come from secrets, everything else is denied. The
|
||||
# same login works on ntfy2.rubenhensen.nl at home.
|
||||
services.ntfy-sh = {
|
||||
enable = true;
|
||||
settings = {
|
||||
base-url = "https://ntfy1.rubenhensen.nl";
|
||||
listen-http = "127.0.0.1:2586";
|
||||
behind-proxy = true;
|
||||
auth-file = "/var/lib/ntfy-sh/user.db";
|
||||
auth-default-access = "deny-all";
|
||||
cache-file = "/var/lib/ntfy-sh/cache.db";
|
||||
# Needed for instant delivery on iOS. ntfy.sh only gets the message
|
||||
# ID, never the content.
|
||||
upstream-base-url = "https://ntfy.sh";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.ntfy-sh.serviceConfig = {
|
||||
LoadCredentialEncrypted = [
|
||||
"ntfy-auth-users:/root/secrets/[%%secrets/ntfy-auth-users%%]"
|
||||
"ntfy-auth-tokens:/root/secrets/[%%secrets/ntfy-auth-tokens%%]"
|
||||
];
|
||||
ExecStart = lib.mkForce (withCredentialEnv "ntfy-sh" {
|
||||
NTFY_AUTH_USERS = "ntfy-auth-users";
|
||||
NTFY_AUTH_TOKENS = "ntfy-auth-tokens";
|
||||
} "${config.services.ntfy-sh.package}/bin/ntfy serve -c /etc/ntfy/server.yml");
|
||||
};
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Gatus (s1.rubenhensen.nl)
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -230,10 +299,34 @@ ${sniMapEntries}
|
||||
{ name = "s2 (homelab)"; link = "https://s2.rubenhensen.nl"; }
|
||||
];
|
||||
};
|
||||
alerting.ntfy = {
|
||||
url = "http://127.0.0.1:2586";
|
||||
topic = "alerts";
|
||||
token = "\${NTFY_TOKEN}";
|
||||
priority = 4;
|
||||
click = "https://s1.rubenhensen.nl";
|
||||
default-alert = {
|
||||
failure-threshold = 3;
|
||||
success-threshold = 2;
|
||||
send-on-resolved = true;
|
||||
};
|
||||
};
|
||||
endpoints = gatusEndpoints;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.gatus = {
|
||||
after = [ "ntfy-sh.service" ];
|
||||
serviceConfig = {
|
||||
LoadCredentialEncrypted = [
|
||||
"ntfy-gatus-token:/root/secrets/[%%secrets/ntfy-gatus-token%%]"
|
||||
];
|
||||
ExecStart = lib.mkForce (withCredentialEnv "gatus" {
|
||||
NTFY_TOKEN = "ntfy-gatus-token";
|
||||
} (lib.getExe config.services.gatus.package));
|
||||
};
|
||||
};
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Stalwart mail server
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ntfy-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: ntfy-auth
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: ntfy-auth
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: NTFY_AUTH_USERS
|
||||
remoteRef:
|
||||
key: kv/ntfy
|
||||
property: auth_users
|
||||
- secretKey: NTFY_AUTH_TOKENS
|
||||
remoteRef:
|
||||
key: kv/ntfy
|
||||
property: auth_tokens
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ntfy-ingress
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: prod-cluster-issuer
|
||||
nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
|
||||
nginx.ingress.kubernetes.io/ssl-passthrough: "false"
|
||||
# Keep subscriptions open, as in the ntfy proxy docs
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "180"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "180"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: ntfy2.rubenhensen.nl
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: ntfy
|
||||
port:
|
||||
number: 80
|
||||
tls:
|
||||
- secretName: letsencrypt-prod
|
||||
hosts:
|
||||
- ntfy2.rubenhensen.nl
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: ntfy
|
||||
|
||||
resources:
|
||||
- external-secret.yaml
|
||||
- data-persistentvolumeclaim.yaml
|
||||
- ntfy-deployment.yaml
|
||||
- ntfy-service.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,66 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ntfy
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: ntfy
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ntfy
|
||||
spec:
|
||||
containers:
|
||||
- name: ntfy
|
||||
image: binwiederhier/ntfy:v2.28.0
|
||||
args: ["serve"]
|
||||
env:
|
||||
- name: NTFY_BASE_URL
|
||||
value: https://ntfy2.rubenhensen.nl
|
||||
- name: NTFY_LISTEN_HTTP
|
||||
value: ":80"
|
||||
- name: NTFY_BEHIND_PROXY
|
||||
value: "true"
|
||||
- name: NTFY_AUTH_FILE
|
||||
value: /var/lib/ntfy/user.db
|
||||
- name: NTFY_AUTH_DEFAULT_ACCESS
|
||||
value: deny-all
|
||||
- name: NTFY_CACHE_FILE
|
||||
value: /var/lib/ntfy/cache.db
|
||||
# Needed for instant delivery on iOS. ntfy.sh only gets the
|
||||
# message ID, never the content.
|
||||
- name: NTFY_UPSTREAM_BASE_URL
|
||||
value: https://ntfy.sh
|
||||
- name: TZ
|
||||
value: Europe/Amsterdam
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: ntfy-auth
|
||||
ports:
|
||||
- containerPort: 80
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v1/health
|
||||
port: 80
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /v1/health
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
memory: "32Mi"
|
||||
cpu: "10m"
|
||||
limits:
|
||||
memory: "256Mi"
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/ntfy
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ntfy-data
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ntfy
|
||||
spec:
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 80
|
||||
selector:
|
||||
app: ntfy
|
||||
+2
-1
@@ -19,7 +19,8 @@
|
||||
"/serpbear/.+\\.yaml$/",
|
||||
"/bolletjestrui/.+\\.yaml$/",
|
||||
"/gitea/.+\\.yaml$/",
|
||||
"/gatus/.+\\.yaml$/"
|
||||
"/gatus/.+\\.yaml$/",
|
||||
"/ntfy/.+\\.yaml$/"
|
||||
]
|
||||
},
|
||||
"customManagers": [
|
||||
|
||||
@@ -5,7 +5,9 @@ metadata:
|
||||
spec:
|
||||
provider:
|
||||
vault:
|
||||
server: "https://vault.rubenhensen.nl"
|
||||
# In-cluster, so syncing secrets doesn't depend on Hetzner or the home
|
||||
# internet connection. Plain HTTP, like the ingress → Vault hop.
|
||||
server: "http://vault-active.vault.svc:8200"
|
||||
path: "kv"
|
||||
# Version is the Vault KV secret engine version.
|
||||
# This can be either "v1" or "v2", defaults to "v2"
|
||||
|
||||
Reference in new issue
Block a user