mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 06:29:09 +01:00
Compare commits
16
Commits
d40560ad56
...
57ea98583a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57ea98583a | ||
|
|
6d9e405126 | ||
|
|
dfdc4451fb | ||
|
|
e19f4b8db5 | ||
|
|
8c2bd89444 | ||
|
|
400d46c462 | ||
|
|
25a6bf3a05 | ||
|
|
a507282103 | ||
|
|
19eacd35f7 | ||
|
|
bb7e2f2bac | ||
|
|
1efa132f9d | ||
|
|
054117ad6d | ||
|
|
ac61c346af | ||
|
|
e23779044f | ||
|
|
53a079dc22 | ||
|
|
3ffb10b627 |
No files matched your search
@@ -36,6 +36,20 @@ spec:
|
||||
|
||||
extraEnvVars:
|
||||
ACTUAL_OPENID_SERVER_HOSTNAME: https://ynab.rubenhensen.nl
|
||||
# Node trusts its built-in roots, which still include the ones
|
||||
# above the compromised eMudhra intermediates. Use patched-ca-bundle
|
||||
# through OpenSSL instead until Node ships a fix.
|
||||
NODE_OPTIONS: --use-openssl-ca
|
||||
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
|
||||
# Service configuration
|
||||
service:
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -16,7 +16,7 @@ spec:
|
||||
sources:
|
||||
- chart: authentik
|
||||
repoURL: https://charts.goauthentik.io
|
||||
targetRevision: 2026.2.1
|
||||
targetRevision: 2026.8.5
|
||||
helm:
|
||||
values: |
|
||||
authentik:
|
||||
@@ -86,8 +86,18 @@ spec:
|
||||
secretKeyRef:
|
||||
name: actualbudget-oidc-client-secret
|
||||
key: client_secret
|
||||
# Point OpenSSL and requests (which uses certifi otherwise) at
|
||||
# patched-ca-bundle, mounted below until the image drops the
|
||||
# compromised eMudhra intermediates.
|
||||
- name: SSL_CERT_FILE
|
||||
value: /etc/ssl/certs/ca-certificates.crt
|
||||
- name: REQUESTS_CA_BUNDLE
|
||||
value: /etc/ssl/certs/ca-certificates.crt
|
||||
server:
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
- name: blueprint-vault
|
||||
configMap:
|
||||
name: authentik-blueprint-vault
|
||||
@@ -131,6 +141,9 @@ spec:
|
||||
- name: blueprint-session
|
||||
mountPath: /blueprints/custom/session-duration.yaml
|
||||
subPath: session-duration.yaml
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
ingress:
|
||||
enabled: true
|
||||
ingressClassName: nginx
|
||||
@@ -144,6 +157,9 @@ spec:
|
||||
- authentik.rubenhensen.nl
|
||||
worker:
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
- name: blueprint-vault
|
||||
configMap:
|
||||
name: authentik-blueprint-vault
|
||||
@@ -187,6 +203,9 @@ spec:
|
||||
- name: blueprint-session
|
||||
mountPath: /blueprints/custom/session-duration.yaml
|
||||
subPath: session-duration.yaml
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
postgresql:
|
||||
enabled: false
|
||||
redis:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: ip-updater
|
||||
name: gatus-folder
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
@@ -10,13 +10,12 @@ spec:
|
||||
source:
|
||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: ip-updater
|
||||
path: gatus
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: ip-updater
|
||||
namespace: gatus
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
prune: true
|
||||
@@ -16,6 +16,10 @@ spec:
|
||||
chart: wordpress
|
||||
helm:
|
||||
values: |
|
||||
# Stopped until PHP is patched (CERT-EU 2026-018, CVE-2026-109618,
|
||||
# CVE-2026-109619). Set back to 1 once the image ships a fixed PHP.
|
||||
replicaCount: 0
|
||||
|
||||
# Official WordPress image configuration
|
||||
image:
|
||||
registry: docker.io
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: ntfy-folder
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: ntfy
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: ntfy
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -9,6 +9,7 @@ resources:
|
||||
- https://raw.githubusercontent.com/argoproj/argo-cd/refs/tags/v2.14.2/manifests/install.yaml
|
||||
- ingress.yaml
|
||||
- oidc-secret.yaml
|
||||
- patched-ca-bundle.yaml
|
||||
|
||||
patches:
|
||||
- target:
|
||||
@@ -18,4 +19,5 @@ patches:
|
||||
name: argocd-notifications-controller
|
||||
path: overlays/argocd-notifications-controller-patch.yaml
|
||||
- path: overlays/argocd-cm.patch.yaml
|
||||
- path: overlays/argocd-rbac-cm.patch.yaml
|
||||
- path: overlays/argocd-rbac-cm.patch.yaml
|
||||
- path: overlays/patched-ca-bundle.patch.yaml
|
||||
@@ -0,0 +1,111 @@
|
||||
# Mount patched-ca-bundle over /etc/ssl/certs so git, helm and the Go
|
||||
# binaries stop trusting the compromised eMudhra intermediates. The whole
|
||||
# directory is replaced because OpenSSL and Go also read the per-root files
|
||||
# next to ca-certificates.crt. Drop this once Argo CD ships a fixed image.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-server
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-repo-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-repo-server
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: argocd-application-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-application-controller
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-applicationset-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-applicationset-controller
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-notifications-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-notifications-controller
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-dex-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: dex
|
||||
volumeMounts:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
name: patched-ca-bundle
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -63,6 +63,25 @@ records:
|
||||
expire: 300
|
||||
type: A
|
||||
content: "46.224.26.65"
|
||||
# Gatus status pages. s2 points straight at home, not through the
|
||||
# Hetzner tunnel, so it stays reachable when Hetzner is down.
|
||||
- name: "s1"
|
||||
expire: 300
|
||||
type: A
|
||||
content: "46.224.26.65"
|
||||
- name: "s2"
|
||||
expire: 300
|
||||
type: A
|
||||
content: "62.41.86.27"
|
||||
# ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
|
||||
- name: "ntfy1"
|
||||
expire: 300
|
||||
type: A
|
||||
content: "46.224.26.65"
|
||||
- name: "ntfy2"
|
||||
expire: 300
|
||||
type: A
|
||||
content: "62.41.86.27"
|
||||
- name: "@"
|
||||
expire: 300
|
||||
type: MX
|
||||
|
||||
@@ -8,7 +8,9 @@ metadata:
|
||||
io.kompose.service: freshrss
|
||||
name: freshrss
|
||||
spec:
|
||||
replicas: 1
|
||||
# Stopped until PHP is patched (CERT-EU 2026-018, CVE-2026-109618,
|
||||
# CVE-2026-109619). Set back to 1 once the image ships a fixed PHP.
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
io.kompose.service: freshrss
|
||||
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
# Gatus
|
||||
|
||||
Two Gatus instances check the same endpoints from two places:
|
||||
|
||||
| Status page | Runs on | Config |
|
||||
|---|---|---|
|
||||
| https://s1.rubenhensen.nl | Hetzner (node001) | `nix-infra-machine/nodes/node001.nix` |
|
||||
| https://s2.rubenhensen.nl | Home cluster | `gatus/config.yaml` (this folder) |
|
||||
|
||||
Each one also checks the other's `/health`. The s2 DNS record points straight
|
||||
at the home IP instead of going through the Hetzner tunnel, so s2 stays
|
||||
reachable when Hetzner is down.
|
||||
|
||||
The endpoint list exists twice, so a new service has to be added in both
|
||||
places. s1 builds its list from `tunneledHosts`, so new tunneled hosts show up
|
||||
there automatically. s2 needs them added by hand.
|
||||
|
||||
## Alerting
|
||||
|
||||
Each Gatus sends alerts to an ntfy instance on its own node:
|
||||
`ntfy1.rubenhensen.nl` (Hetzner, node001.nix) and `ntfy2.rubenhensen.nl`
|
||||
(home, `ntfy/` in this repo). Subscribe the phone app to topic `alerts` on
|
||||
both servers, with the same login on each.
|
||||
|
||||
Only the watcher that sees a service from outside alerts on it:
|
||||
|
||||
- s1 alerts on everything that runs at home (all tunneled apps, Scouting MBG)
|
||||
and on s2.
|
||||
- s2 alerts on mail and on s1.
|
||||
|
||||
An outage on either node still reaches you, through the other node's ntfy,
|
||||
and you get one notification per outage instead of two. An alert fires after
|
||||
3 failed checks in a row (about 3 minutes), and you get another notification
|
||||
when the service recovers.
|
||||
|
||||
Both ntfy servers deny everything except the provisioned user. The
|
||||
credentials are the same on both:
|
||||
|
||||
- Home: Vault `kv/ntfy` with `auth_users`, `auth_tokens` and `gatus_token`,
|
||||
pulled in by External Secrets.
|
||||
- Hetzner: the nix-infra secrets `ntfy-auth-users`, `ntfy-auth-tokens` and
|
||||
`ntfy-gatus-token`.
|
||||
|
||||
## Reading the two pages
|
||||
|
||||
- s1 sees the homelab from outside, so trust it for homelab services. s2 sees
|
||||
Hetzner from outside, so trust it for mail and anything else on Hetzner.
|
||||
- Every `*.rubenhensen.nl` app is tunneled Hetzner → home, so a Hetzner
|
||||
outage shows all of them as down on both pages. That is accurate: nobody
|
||||
can reach them then.
|
||||
- If s1 and s2 disagree about an endpoint, the problem is the network path
|
||||
between one watcher and that service, not the service itself.
|
||||
- Neither page has history for the time its own node was down.
|
||||
|
||||
## Later: one URL with self-hosted DNS failover
|
||||
|
||||
This is the plan for replacing s1/s2 with a single `status.rubenhensen.nl`
|
||||
that always points at a watcher that's up, with no third party involved.
|
||||
|
||||
Hand `status.rubenhensen.nl` over to two nameservers we run ourselves, one per
|
||||
node, both PowerDNS Authoritative. PowerDNS LUA records health-check
|
||||
addresses and only return the healthy ones:
|
||||
|
||||
```
|
||||
@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.27'}})"
|
||||
```
|
||||
|
||||
The double braces make two groups. PowerDNS returns the first group that has
|
||||
a healthy address, so Hetzner is primary and home is the fallback. Each
|
||||
nameserver runs the check from its own location.
|
||||
|
||||
| Situation | Hetzner NS | Home NS | Visitors end up on |
|
||||
|---|---|---|---|
|
||||
| Both up | answers Hetzner | answers Hetzner | Hetzner |
|
||||
| Hetzner down | dead | Hetzner check fails, answers home | home |
|
||||
| Home down / home internet down | answers Hetzner | dead | Hetzner |
|
||||
| Hetzner up, its Gatus broken | own check fails, answers home | answers home | home |
|
||||
|
||||
A node that goes down takes its nameserver down with it, so DNS resolvers
|
||||
switch to the other nameserver. The two nodes never have to agree with each
|
||||
other: the answer depends on what the internet can reach. The 30s TTL caps
|
||||
how long clients keep a stale answer.
|
||||
|
||||
Steps:
|
||||
|
||||
1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add
|
||||
`status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`,
|
||||
plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.27).
|
||||
First check that TransIP accepts NS records for a subdomain.
|
||||
2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and
|
||||
the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp.
|
||||
3. Home: run PowerDNS in the cluster with the same zone file, on 53/udp and
|
||||
53/tcp of the home IP (router port forward, and check how the cluster
|
||||
exposes UDP).
|
||||
4. Both Gatus instances also serve `status.rubenhensen.nl`. On Hetzner, add the
|
||||
name to the SNI map, the same way as s1.
|
||||
5. TLS: Hetzner gets its cert through HTTP-01 as usual. On Hetzner's :80 vhost
|
||||
for `status`, serve local ACME challenges and proxy the rest to the home
|
||||
cluster (`try_files $uri @home`). That way cert-manager's HTTP-01 also
|
||||
works while DNS points at Hetzner. When Hetzner is down, DNS points home
|
||||
and cert-manager reaches the home cluster directly.
|
||||
6. Requirements: a static home IP (`homeIP` in node001.nix already assumes
|
||||
one), and port 53 open on both nodes.
|
||||
|
||||
After that, s1/s2 can be removed, or kept as direct links to each instance.
|
||||
@@ -0,0 +1,146 @@
|
||||
# s2.rubenhensen.nl: Gatus on the home cluster.
|
||||
# Its twin s1.rubenhensen.nl runs on the Hetzner box, see
|
||||
# nix-infra-machine/nodes/node001.nix. Keep the endpoint lists in sync.
|
||||
storage:
|
||||
type: sqlite
|
||||
path: /data/data.db
|
||||
|
||||
ui:
|
||||
title: Status (homelab)
|
||||
header: s2 · homelab
|
||||
buttons:
|
||||
- name: s1 (Hetzner)
|
||||
link: https://s1.rubenhensen.nl
|
||||
|
||||
# NTFY_TOKEN comes from the gatus-ntfy secret (Vault kv/ntfy)
|
||||
alerting:
|
||||
ntfy:
|
||||
url: http://ntfy.ntfy.svc.cluster.local
|
||||
topic: alerts
|
||||
token: ${NTFY_TOKEN}
|
||||
priority: 4
|
||||
click: https://s2.rubenhensen.nl
|
||||
default-alert:
|
||||
failure-threshold: 3
|
||||
success-threshold: 2
|
||||
send-on-resolved: true
|
||||
|
||||
# s2 is the outside view of Hetzner, so it alerts on mail and on s1. The
|
||||
# home services are left to s1.
|
||||
endpoints:
|
||||
# Tunneled through Hetzner to the home cluster
|
||||
- name: rss
|
||||
group: rubenhensen.nl
|
||||
url: https://rss.rubenhensen.nl
|
||||
interval: 60s
|
||||
# mod_auth_openidc answers 401 instead of redirecting to the login
|
||||
# page unless the client asks for HTML
|
||||
headers:
|
||||
Accept: text/html
|
||||
conditions: &http-ok
|
||||
- "[STATUS] == 200"
|
||||
- "[CERTIFICATE_EXPIRATION] > 168h"
|
||||
- name: authentik
|
||||
group: rubenhensen.nl
|
||||
url: https://authentik.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: vault
|
||||
group: rubenhensen.nl
|
||||
url: https://vault.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: ynab
|
||||
group: rubenhensen.nl
|
||||
url: https://ynab.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: argocd
|
||||
group: rubenhensen.nl
|
||||
url: https://argocd.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: ha
|
||||
group: rubenhensen.nl
|
||||
url: https://ha.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: longhorn
|
||||
group: rubenhensen.nl
|
||||
url: https://longhorn.rubenhensen.nl
|
||||
interval: 60s
|
||||
# Behind basic auth, so 401 means it's up
|
||||
conditions:
|
||||
- "[STATUS] == 401"
|
||||
- "[CERTIFICATE_EXPIRATION] > 168h"
|
||||
- name: lingo
|
||||
group: rubenhensen.nl
|
||||
url: https://lingo.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: blog
|
||||
group: rubenhensen.nl
|
||||
url: https://blog.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: serpbear
|
||||
group: rubenhensen.nl
|
||||
url: https://serpbear.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: git
|
||||
group: rubenhensen.nl
|
||||
url: https://git.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: vic
|
||||
group: rubenhensen.nl
|
||||
url: https://vic.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
|
||||
# Stalwart on Hetzner
|
||||
- name: webmail
|
||||
group: mail
|
||||
url: https://mail.rubenhensen.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
alerts: &ntfy
|
||||
- type: ntfy
|
||||
- name: smtp
|
||||
group: mail
|
||||
url: tcp://mail.rubenhensen.nl:25
|
||||
interval: 60s
|
||||
conditions:
|
||||
- "[CONNECTED] == true"
|
||||
alerts: *ntfy
|
||||
- name: imaps
|
||||
group: mail
|
||||
url: tls://mail.rubenhensen.nl:993
|
||||
interval: 60s
|
||||
conditions:
|
||||
- "[CONNECTED] == true"
|
||||
- "[CERTIFICATE_EXPIRATION] > 168h"
|
||||
alerts: *ntfy
|
||||
|
||||
# Served straight from the home IP
|
||||
- name: scoutingmbg.nl
|
||||
group: Scouting MBG
|
||||
url: https://scoutingmbg.nl
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
- name: foto.scoutingmbg.nl
|
||||
group: Scouting MBG
|
||||
url: https://foto.scoutingmbg.nl/_app
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
|
||||
# The other watcher
|
||||
- name: s1 (Hetzner)
|
||||
group: watchers
|
||||
url: https://s1.rubenhensen.nl/health
|
||||
interval: 60s
|
||||
conditions:
|
||||
- "[STATUS] == 200"
|
||||
- "[BODY].status == UP"
|
||||
alerts: *ntfy
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: gatus-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
@@ -1,17 +1,17 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: ip-updater-ssh-key
|
||||
name: gatus-ntfy
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: ip-updater-ssh-key
|
||||
name: gatus-ntfy
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: private_key
|
||||
- secretKey: NTFY_TOKEN
|
||||
remoteRef:
|
||||
key: kv/ip-updater
|
||||
property: ssh_private_key
|
||||
key: kv/ntfy
|
||||
property: gatus_token
|
||||
@@ -0,0 +1,57 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: gatus
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: gatus
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: gatus
|
||||
spec:
|
||||
containers:
|
||||
- name: gatus
|
||||
image: twinproduction/gatus:v5.37.0
|
||||
env:
|
||||
- name: GATUS_CONFIG_PATH
|
||||
value: /config/config.yaml
|
||||
- name: TZ
|
||||
value: Europe/Amsterdam
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: gatus-ntfy
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
resources:
|
||||
requests:
|
||||
memory: "64Mi"
|
||||
cpu: "20m"
|
||||
limits:
|
||||
memory: "256Mi"
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
readOnly: true
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: gatus-config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: gatus-data
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: gatus
|
||||
spec:
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
selector:
|
||||
app: gatus
|
||||
@@ -0,0 +1,25 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: gatus-ingress
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: prod-cluster-issuer
|
||||
nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
|
||||
nginx.ingress.kubernetes.io/ssl-passthrough: "false"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: s2.rubenhensen.nl
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gatus
|
||||
port:
|
||||
number: 8080
|
||||
tls:
|
||||
- secretName: letsencrypt-prod
|
||||
hosts:
|
||||
- s2.rubenhensen.nl
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: gatus
|
||||
|
||||
resources:
|
||||
- external-secret.yaml
|
||||
- data-persistentvolumeclaim.yaml
|
||||
- gatus-deployment.yaml
|
||||
- gatus-service.yaml
|
||||
- ingress.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: gatus-config
|
||||
files:
|
||||
- config.yaml
|
||||
@@ -1,39 +0,0 @@
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: ip-updater
|
||||
spec:
|
||||
schedule: "*/5 * * * *"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: ip-updater
|
||||
image: alpine:3.19
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
apk add --no-cache openssh-client curl > /dev/null 2>&1
|
||||
cp /ssh-key/private_key /tmp/ssh_key
|
||||
chmod 600 /tmp/ssh_key
|
||||
IP=$(curl -s --max-time 10 https://api.ipify.org)
|
||||
if [ -z "$IP" ]; then
|
||||
echo "Failed to get external IP"
|
||||
exit 1
|
||||
fi
|
||||
echo "Current external IP: $IP"
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-i /tmp/ssh_key root@46.224.26.65 "$IP"
|
||||
volumeMounts:
|
||||
- name: ssh-key
|
||||
mountPath: /ssh-key
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: ssh-key
|
||||
secret:
|
||||
secretName: ip-updater-ssh-key
|
||||
defaultMode: 0400
|
||||
restartPolicy: OnFailure
|
||||
@@ -38,6 +38,75 @@ let
|
||||
locations."/".proxyPass = "http://${homeIP}";
|
||||
};
|
||||
}) tunneledHosts);
|
||||
|
||||
# Gatus endpoints for s1.rubenhensen.nl. Its twin s2.rubenhensen.nl runs
|
||||
# on the home cluster with the same list, see gatus/config.yaml in k8scd.
|
||||
# Keep the two in sync.
|
||||
gatusHttp = group: name: url: {
|
||||
inherit name group url;
|
||||
interval = "60s";
|
||||
conditions = [
|
||||
"[STATUS] == 200"
|
||||
"[CERTIFICATE_EXPIRATION] > 168h"
|
||||
];
|
||||
};
|
||||
|
||||
# Per-host tweaks on top of the default check.
|
||||
gatusOverrides = {
|
||||
# Behind basic auth, so 401 means it's up
|
||||
"longhorn.rubenhensen.nl".conditions = [
|
||||
"[STATUS] == 401"
|
||||
"[CERTIFICATE_EXPIRATION] > 168h"
|
||||
];
|
||||
# mod_auth_openidc answers 401 instead of redirecting to the login
|
||||
# page unless the client asks for HTML
|
||||
"rss.rubenhensen.nl".headers.Accept = "text/html";
|
||||
};
|
||||
|
||||
# s1 is the outside view of home, so it alerts on what runs there and on
|
||||
# s2. Mail lives on this box, so s2 alerts on that.
|
||||
gatusAlert = { alerts = [ { type = "ntfy"; } ]; };
|
||||
|
||||
gatusEndpoints =
|
||||
map (h:
|
||||
gatusHttp "rubenhensen.nl" (lib.removeSuffix ".rubenhensen.nl" h) "https://${h}"
|
||||
// gatusAlert
|
||||
// (gatusOverrides.${h} or { })) tunneledHosts
|
||||
++ [
|
||||
(gatusHttp "mail" "webmail" "https://mail.rubenhensen.nl")
|
||||
{
|
||||
name = "smtp";
|
||||
group = "mail";
|
||||
url = "tcp://mail.rubenhensen.nl:25";
|
||||
interval = "60s";
|
||||
conditions = [ "[CONNECTED] == true" ];
|
||||
}
|
||||
{
|
||||
name = "imaps";
|
||||
group = "mail";
|
||||
url = "tls://mail.rubenhensen.nl:993";
|
||||
interval = "60s";
|
||||
conditions = [ "[CONNECTED] == true" "[CERTIFICATE_EXPIRATION] > 168h" ];
|
||||
}
|
||||
(gatusHttp "Scouting MBG" "scoutingmbg.nl" "https://scoutingmbg.nl" // gatusAlert)
|
||||
(gatusHttp "Scouting MBG" "foto.scoutingmbg.nl" "https://foto.scoutingmbg.nl/_app" // gatusAlert)
|
||||
({
|
||||
name = "s2 (homelab)";
|
||||
group = "watchers";
|
||||
url = "https://s2.rubenhensen.nl/health";
|
||||
interval = "60s";
|
||||
conditions = [ "[STATUS] == 200" "[BODY].status == UP" ];
|
||||
} // gatusAlert)
|
||||
];
|
||||
|
||||
# Runs a service with secrets from LoadCredentialEncrypted exported as
|
||||
# env vars, for services that only take secrets through the environment.
|
||||
withCredentialEnv = name: vars: cmd: pkgs.writeShellScript "${name}-with-credentials" ''
|
||||
${lib.concatStrings (lib.mapAttrsToList (var: cred: ''
|
||||
export ${var}="$(cat "$CREDENTIALS_DIRECTORY/${cred}")"
|
||||
'') vars)}
|
||||
exec ${cmd}
|
||||
'';
|
||||
in
|
||||
{
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -53,6 +122,17 @@ in
|
||||
80 # HTTP (ACME)
|
||||
];
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Trust store
|
||||
# ──────────────────────────────────────────────
|
||||
# Distrust the roots above the compromised eMudhra intermediates until
|
||||
# nixpkgs ships NSS 3.131.1, see https://emudhra.we-have-a-problem.com.
|
||||
# Everything this box talks to chains to ISRG Root X1.
|
||||
security.pki.caCertificateBlacklist = [
|
||||
"Comodo AAA Services root"
|
||||
"emSign ECC Root CA - G3"
|
||||
];
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# ACME / Let's Encrypt
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -86,9 +166,12 @@ in
|
||||
# SNI passthrough on 443:
|
||||
# - mail.rubenhensen.nl (and anything else) → local stalwart on 8443
|
||||
# - tunneled hosts → home cluster ingress on 443
|
||||
# - s1 / ntfy1.rubenhensen.nl → local nginx on 8444 (Gatus, ntfy)
|
||||
streamConfig = ''
|
||||
map $ssl_preread_server_name $tunnel_upstream {
|
||||
${sniMapEntries}
|
||||
s1.rubenhensen.nl 127.0.0.1:8444;
|
||||
ntfy1.rubenhensen.nl 127.0.0.1:8444;
|
||||
default 127.0.0.1:8443;
|
||||
}
|
||||
|
||||
@@ -137,6 +220,121 @@ ${sniMapEntries}
|
||||
locations."/.well-known/acme-challenge/".root = "/var/lib/acme/acme-challenge";
|
||||
locations."/".return = "301 https://$host$request_uri";
|
||||
};
|
||||
# Gatus status page. The stream block above passes its SNI to
|
||||
# 8444, where nginx terminates TLS with its own ACME cert.
|
||||
"s1.rubenhensen.nl" = {
|
||||
listen = [
|
||||
{ addr = "0.0.0.0"; port = 80; }
|
||||
{ addr = "[::]"; port = 80; }
|
||||
{ addr = "127.0.0.1"; port = 8444; ssl = true; }
|
||||
];
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/".proxyPass = "http://127.0.0.1:8090";
|
||||
};
|
||||
# ntfy for Gatus alerts, same TLS setup as s1. Proxy settings follow
|
||||
# the ntfy docs so subscriptions can stay open.
|
||||
"ntfy1.rubenhensen.nl" = {
|
||||
listen = [
|
||||
{ addr = "0.0.0.0"; port = 80; }
|
||||
{ addr = "[::]"; port = 80; }
|
||||
{ addr = "127.0.0.1"; port = 8444; ssl = true; }
|
||||
];
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:2586";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = ''
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
proxy_connect_timeout 3m;
|
||||
proxy_send_timeout 3m;
|
||||
proxy_read_timeout 3m;
|
||||
client_max_body_size 0;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# ntfy (ntfy1.rubenhensen.nl)
|
||||
# ──────────────────────────────────────────────
|
||||
# Users and tokens come from secrets, everything else is denied. The
|
||||
# same login works on ntfy2.rubenhensen.nl at home.
|
||||
services.ntfy-sh = {
|
||||
enable = true;
|
||||
settings = {
|
||||
base-url = "https://ntfy1.rubenhensen.nl";
|
||||
listen-http = "127.0.0.1:2586";
|
||||
behind-proxy = true;
|
||||
auth-file = "/var/lib/ntfy-sh/user.db";
|
||||
auth-default-access = "deny-all";
|
||||
cache-file = "/var/lib/ntfy-sh/cache.db";
|
||||
# Needed for instant delivery on iOS. ntfy.sh only gets the message
|
||||
# ID, never the content.
|
||||
upstream-base-url = "https://ntfy.sh";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.ntfy-sh.serviceConfig = {
|
||||
LoadCredentialEncrypted = [
|
||||
"ntfy-auth-users:/root/secrets/[%%secrets/ntfy-auth-users%%]"
|
||||
"ntfy-auth-tokens:/root/secrets/[%%secrets/ntfy-auth-tokens%%]"
|
||||
];
|
||||
ExecStart = lib.mkForce (withCredentialEnv "ntfy-sh" {
|
||||
NTFY_AUTH_USERS = "ntfy-auth-users";
|
||||
NTFY_AUTH_TOKENS = "ntfy-auth-tokens";
|
||||
} "${config.services.ntfy-sh.package}/bin/ntfy serve -c /etc/ntfy/server.yml");
|
||||
};
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Gatus (s1.rubenhensen.nl)
|
||||
# ──────────────────────────────────────────────
|
||||
services.gatus = {
|
||||
enable = true;
|
||||
settings = {
|
||||
web = {
|
||||
address = "127.0.0.1";
|
||||
port = 8090;
|
||||
};
|
||||
storage = {
|
||||
type = "sqlite";
|
||||
path = "/var/lib/gatus/data.db";
|
||||
};
|
||||
ui = {
|
||||
title = "Status (Hetzner)";
|
||||
header = "s1 · Hetzner";
|
||||
buttons = [
|
||||
{ name = "s2 (homelab)"; link = "https://s2.rubenhensen.nl"; }
|
||||
];
|
||||
};
|
||||
alerting.ntfy = {
|
||||
url = "http://127.0.0.1:2586";
|
||||
topic = "alerts";
|
||||
token = "\${NTFY_TOKEN}";
|
||||
priority = 4;
|
||||
click = "https://s1.rubenhensen.nl";
|
||||
default-alert = {
|
||||
failure-threshold = 3;
|
||||
success-threshold = 2;
|
||||
send-on-resolved = true;
|
||||
};
|
||||
};
|
||||
endpoints = gatusEndpoints;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.gatus = {
|
||||
after = [ "ntfy-sh.service" ];
|
||||
serviceConfig = {
|
||||
LoadCredentialEncrypted = [
|
||||
"ntfy-gatus-token:/root/secrets/[%%secrets/ntfy-gatus-token%%]"
|
||||
];
|
||||
ExecStart = lib.mkForce (withCredentialEnv "gatus" {
|
||||
NTFY_TOKEN = "ntfy-gatus-token";
|
||||
} (lib.getExe config.services.gatus.package));
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ntfy-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: ntfy-auth
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: ntfy-auth
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: NTFY_AUTH_USERS
|
||||
remoteRef:
|
||||
key: kv/ntfy
|
||||
property: auth_users
|
||||
- secretKey: NTFY_AUTH_TOKENS
|
||||
remoteRef:
|
||||
key: kv/ntfy
|
||||
property: auth_tokens
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ntfy-ingress
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: prod-cluster-issuer
|
||||
nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
|
||||
nginx.ingress.kubernetes.io/ssl-passthrough: "false"
|
||||
# Keep subscriptions open, as in the ntfy proxy docs
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "180"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "180"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: ntfy2.rubenhensen.nl
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: ntfy
|
||||
port:
|
||||
number: 80
|
||||
tls:
|
||||
- secretName: letsencrypt-prod
|
||||
hosts:
|
||||
- ntfy2.rubenhensen.nl
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: ntfy
|
||||
|
||||
resources:
|
||||
- external-secret.yaml
|
||||
- data-persistentvolumeclaim.yaml
|
||||
- ntfy-deployment.yaml
|
||||
- ntfy-service.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,66 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ntfy
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: ntfy
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ntfy
|
||||
spec:
|
||||
containers:
|
||||
- name: ntfy
|
||||
image: binwiederhier/ntfy:v2.28.0
|
||||
args: ["serve"]
|
||||
env:
|
||||
- name: NTFY_BASE_URL
|
||||
value: https://ntfy2.rubenhensen.nl
|
||||
- name: NTFY_LISTEN_HTTP
|
||||
value: ":80"
|
||||
- name: NTFY_BEHIND_PROXY
|
||||
value: "true"
|
||||
- name: NTFY_AUTH_FILE
|
||||
value: /var/lib/ntfy/user.db
|
||||
- name: NTFY_AUTH_DEFAULT_ACCESS
|
||||
value: deny-all
|
||||
- name: NTFY_CACHE_FILE
|
||||
value: /var/lib/ntfy/cache.db
|
||||
# Needed for instant delivery on iOS. ntfy.sh only gets the
|
||||
# message ID, never the content.
|
||||
- name: NTFY_UPSTREAM_BASE_URL
|
||||
value: https://ntfy.sh
|
||||
- name: TZ
|
||||
value: Europe/Amsterdam
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: ntfy-auth
|
||||
ports:
|
||||
- containerPort: 80
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v1/health
|
||||
port: 80
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /v1/health
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
memory: "32Mi"
|
||||
cpu: "10m"
|
||||
limits:
|
||||
memory: "256Mi"
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/ntfy
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ntfy-data
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ntfy
|
||||
spec:
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 80
|
||||
selector:
|
||||
app: ntfy
|
||||
+3
-1
@@ -18,7 +18,9 @@
|
||||
"/apps/templates/.+\\.yaml$/",
|
||||
"/serpbear/.+\\.yaml$/",
|
||||
"/bolletjestrui/.+\\.yaml$/",
|
||||
"/gitea/.+\\.yaml$/"
|
||||
"/gitea/.+\\.yaml$/",
|
||||
"/gatus/.+\\.yaml$/",
|
||||
"/ntfy/.+\\.yaml$/"
|
||||
]
|
||||
},
|
||||
"customManagers": [
|
||||
|
||||
@@ -5,7 +5,9 @@ metadata:
|
||||
spec:
|
||||
provider:
|
||||
vault:
|
||||
server: "https://vault.rubenhensen.nl"
|
||||
# In-cluster, so syncing secrets doesn't depend on Hetzner or the home
|
||||
# internet connection. Plain HTTP, like the ingress → Vault hop.
|
||||
server: "http://vault-active.vault.svc:8200"
|
||||
path: "kv"
|
||||
# Version is the Vault KV secret engine version.
|
||||
# This can be either "v1" or "v2", defaults to "v2"
|
||||
|
||||
@@ -11,6 +11,11 @@ spec:
|
||||
targetRevision: 8.4.0
|
||||
helm:
|
||||
values: |
|
||||
# The CRD upgrade hook pulls docker.io/bitnami/kubectl, which Bitnami
|
||||
# has removed, so every sync hangs on it. The CRDs only change with
|
||||
# the chart version; turn this back on (with another kubectl image)
|
||||
# when bumping targetRevision.
|
||||
upgradeCRDs: false
|
||||
initContainers:
|
||||
- name: velero-plugin-for-aws
|
||||
image: velero/velero-plugin-for-aws:v1.11.1
|
||||
@@ -43,13 +48,15 @@ spec:
|
||||
nodeAgent:
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
# 512Mi got the server OOMKilled 314 times, including halfway
|
||||
# through a backup of ~14.7k items.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
memory: 2Gi
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
|
||||
Reference in new issue
Block a user