renovate: patch updates of low-risk apps merge themselves

Pass the visitor's address from node001 to the home cluster

apps/templates/cert-manager-helm.yaml

@@ -12,7 +12,7 @@ spec:
   project: default
   source:
     repoURL: https://charts.jetstack.io
-    targetRevision: v1.21.2
+    targetRevision: v1.21.3
     chart: cert-manager
     helm:
       parameters:

apps/templates/external-secrets-helm.yaml

@@ -17,7 +17,7 @@ spec:
   sources:
     - chart: external-secrets
       repoURL: https://charts.external-secrets.io
-      targetRevision: 2.13.0
+      targetRevision: 2.14.0
       helm:
         values: |
           resources:

apps/templates/mbgwp-official-helm.yaml

@@ -12,7 +12,7 @@ spec:
   project: default
   source:
     repoURL: https://groundhog2k.github.io/helm-charts/
-    targetRevision: 0.15.0
+    targetRevision: 0.16.7
     chart: wordpress
     helm:
       values: |
@@ -101,6 +101,23 @@ spec:
           timeoutSeconds: 5
           failureThreshold: 30
 
+        # apache2.conf includes ports.conf after mods-enabled/*.conf, so the
+        # prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
+        # 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
+        # crawler asking for 88 pages at once got the container OOMKilled on
+        # 2026-10-25. With 12 workers a burst waits in the listen backlog
+        # instead. Listen is the chart's default (containerPort).
+        apachePortsConfig: |
+          Listen 8000
+          <IfModule mpm_prefork_module>
+            StartServers            4
+            MinSpareServers         2
+            MaxSpareServers         6
+            ServerLimit             12
+            MaxRequestWorkers       12
+            MaxConnectionsPerChild  1000
+          </IfModule>
+
         # Custom PHP configuration
         customPhpConfig: |
           upload_max_filesize = 64M

apps/templates/snapshot-controller-helm.yaml

@@ -7,7 +7,7 @@ spec:
   project: default
   source:
     repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
-    targetRevision: v8.2.1
+    targetRevision: v8.7.0
     path: deploy/kubernetes/snapshot-controller
     kustomize:
       patches:

backup-testing/restore-test.sh

@@ -3,7 +3,7 @@
 # k3s in Docker, to prove the backups can be restored. Nothing on the NAS
 # changes:
 # - one backup (backups/<name>/) and the kopia repositories of the chosen
-#   namespaces (kopia/<namespace>/) are copied read-only from MinIO with
+#   namespaces (kopia/<namespace>/) are copied read-only from the NAS with
 #   rclone into a Docker volume. A second Velero on the real bucket could
 #   run kopia maintenance on the same repositories as the cluster's own
 # - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
@@ -19,7 +19,7 @@
 # Defaults: the newest Completed daily-backup, and ntfy freshrss
 # actual-budget. Not gatus: a restored Gatus would start sending alerts.
 #
-# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
+# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
 # into a temporary directory, and the copied backup holds every Secret of
 # the cluster; all of it is deleted at the end. KEEP=1 keeps the test
 # cluster (kubeconfig in the printed directory) until you run it with
@@ -33,8 +33,6 @@
 set -euo pipefail
 
 PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
-MINIO=${MINIO:-http://192.168.1.99:9000}
-BUCKET=velero-backups
 PORT=${PORT:-16443}
 K3S=rancher/k3s:v1.34.9-k3s1
 RCLONE=rclone/rclone:1.75.1
@@ -73,6 +71,18 @@ wait_for() {
   done
 }
 
+# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
+# request signed for the wrong region (rclone defaults to us-east-1); MinIO
+# did so with a bare 400 on HEAD.
+if [[ -n ${FROM_VOLUME:-} ]]; then
+  BUCKET=${BUCKET:-velero-backups}
+else
+  bsl=$(pk -n velero get backupstoragelocations default -o json)
+  S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
+  REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
+  BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
+fi
+
 backup=${1:-}
 shift || true
 namespaces=("$@")
@@ -91,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
   docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
 else
 echo "== copy from the NAS (read-only)"
-(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
+(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
   awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
                  /aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
-  printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env")
+  printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
 copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
 copy "backups/$backup"
 for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done

docs/immich-scouts-online.md

@@ -123,15 +123,13 @@ D. auth.scoutingmbg.nl (2026-10-25)
 
 ## Open points
 
-- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
-  foto.scoutingmbg.nl from a CNAME to `mbg.hensen.io` (the home IP) to a
-  CNAME to `rubenhensen.nl`, which is node001 on Hetzner. Only ns2 has it
-  (serial 2026102471); ns1 is stuck on serial 2026060640 from June and
-  still answers `mbg.hensen.io`. node001 now tunnels foto.scoutingmbg.nl to
-  the home cluster like its other hosts, so both answers work. Keep
-  hensen.io until ns1 has caught up; ask the DNS host to fix ns1, and
-  decide whether foto should stay behind node001 or point straight home
-  (CNAME `scoutingmbg.nl`)
+- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
+  scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
+  all the records it moved, and node001 tunnels all four home. ns1 lagged
+  behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
+  uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
+  `a` now means node001, which is fine: the WordPress site sends no mail
+  and the group's mail goes through Google (`mx`)
 - Public shared links become public for real once sol-auth is gone.
 - Photos leiding back up from their phones stay private until they add
   them to a shared album.

docs/longhorn-second-ssd.md

@@ -38,14 +38,15 @@ between them live.
 
 ## Before: shrink what has to move
 
-- [ ] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
+- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
   snapshots from before October 2026, nearly all on immich-claim
   (pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
   2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
   longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
   don't depend on them. Deleting a snapshot merges it into the next one,
   which writes a lot, so do this in a quiet hour, one volume at a time.
-  Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
+  Happened on 2026-10-24, from a Claude Code session at Ruben's request
+  (the 45 snapshots from before October 2026, small volumes first): immich-claim's
   old snapshots were removed at 14:43 UTC and Longhorn merged them one
   after another, about 45 minutes each, until 23:15. For those 8.5 hours
   sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on

docs/roadmap.md

@@ -10,10 +10,11 @@ One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
 SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
 35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
 volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
-NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
-01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
-and s2 (home) watch each other and alert through ntfy; s2 also gets an
-hourly report on the backups and Vault's seal from inside the cluster.
+NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to Garage
+(MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90 days).
+Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
+through ntfy; s2 also gets an hourly report from inside the cluster on the
+backups, Vault's seal, the Argo CD apps and the CNPG clusters.
 Every app namespace has NetworkPolicies, and CI renders and validates all
 apps on every PR. node001 runs mail and tunnels the public hosts home.
 
@@ -33,8 +34,9 @@ docs/longhorn-second-ssd.md:
 - [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
   (`backend-*.tar`, `dqlite-pre-*`)
 - [ ] Log in to immich as a jeugdlid (quota 0)
-- [ ] Back to 1.36/stable once it is at v1.36.5 or later. The snap is held,
-  so microk8s gets no updates at all until someone refreshes it
+- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
+  2026-10-25). The snap is held, so microk8s gets no updates at all until
+  someone refreshes it
 - [ ] Second SSD, docs/longhorn-second-ssd.md
 - [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
 
@@ -48,8 +50,9 @@ group's ns2: ns1 is stuck on an old serial and still answers
 mbg.hensen.io (docs/immich-scouts-online.md).
 
 - [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
-- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
-  rubenhensen.nl too
+- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
+  `rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
+  cluster uses hensen.io anymore. hensen.io can expire
 
 ### 2. Off-site backups
 
@@ -60,24 +63,25 @@ belong to the group.
 There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
 friend.
 
-- [ ] Ours to his bucket: Hyper Backup on the Synology, destination "S3
-  Storage" with his endpoint as a custom server. Client-side encryption
-  on: the kopia repositories use Velero's default password
-  (`static-passw0rd`), so without it he could read everything, the
-  group's photos included. Keep the encryption key next to the Vault
-  unseal key. Source: the shared folder with MinIO's `velero-backups`
-  bucket (~150-200 GB). Daily after the Velero backup, Smart Recycle,
-  mail on failure
+- [ ] Ours to his bucket: `rclone sync` from Garage's `velero-backups`
+  (~190 GB) to his bucket through an rclone `crypt` remote, in Docker on
+  the NAS like the copy from MinIO (see 5). Encrypted on our side: the
+  kopia repositories use Velero's default password (`static-passw0rd`),
+  so without it he could read everything, the group's photos included.
+  Keep the crypt passwords next to the Vault unseal key. Daily after the
+  Velero backup, ntfy on failure. Hyper Backup, the earlier plan, backs up
+  shared folders; Garage keeps its objects as blocks plus a metadata
+  database, and a copy of that folder taken while Garage runs is not
+  consistent
 - [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
   now, 500 GB to grow), and versioning or object lock on the bucket, so
   whoever takes over the NAS can't delete the off-site copy as well
-- [ ] His backups with us: Garage (not MinIO, see 5) in Container Manager
-  on the Synology, in its own shared folder, with a bucket and key for him
-  and a quota. Only the S3 API public, as `s3.rubenhensen.nl` through
-  node001 and Envoy with a cert-manager certificate; the admin API stays
-  on the LAN. The cluster side (DNS, node001 tunnel, Certificate,
-  HTTPRoute) goes in git. Needed: how much space he wants, free space on
-  the NAS
+- [ ] His backups with us: a bucket and key for him, with a quota, in the
+  Garage on the NAS (`nas/garage`, see 5). Only the S3 API public, as
+  `s3.rubenhensen.nl` through node001 and Envoy with a cert-manager
+  certificate; the admin API stays on the LAN. The cluster side (DNS,
+  node001 tunnel, Certificate, HTTPRoute) goes in git. Needed: how much
+  space he wants, free space on the NAS
 - [ ] Decide which copy of the immich library goes off-site (see 5)
 - [ ] A restore test that runs on a schedule. `backup-testing/` predates
   Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
@@ -97,8 +101,20 @@ friend.
   ReadOnly location: data restored byte for byte (same sha256), deployment
   available, everything removed afterwards. Two pitfalls found on the way
   are in the script's header. Not yet run against the real backups:
-- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from
-  the cluster), then schedule it
+- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
+  daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
+  volume came back with its backed-up byte count (actual-budget 8.4 MB,
+  freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
+  first run failed on a 400 from MinIO: rclone signed for us-east-1 while
+  MinIO runs region `minio`; the script now reads endpoint, region and
+  bucket from Velero's location. Copying freshrss's kopia repository took
+  14 of the ~17 minutes
+- [x] Ran it again on 2026-10-26 against the copy in Garage
+  (daily-backup-20261026010028, same namespaces), before Velero switched
+  over (see 5): passed, in 5 minutes instead of 17
+- [ ] Run it on a schedule. It needs Docker and an S3 key, so not in the
+  cluster itself: the Mac (launchd) or node001, ideally with a read-only
+  Garage key (`garage key create`, `garage bucket allow --read`)
 
 ### 3. authentik 2026.2.1 → 2026.8.5
 
@@ -153,8 +169,8 @@ unseals it.
   - The OIDC job called `vault auth tune -listing_visibility`, a flag that
     doesn't exist, so the OIDC mount's listing visibility was never set.
     Fixed; the job now uses the 2.2.1 CLI instead of 1.15
-- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
-  places outside this cluster; without it the data in Vault is lost
+- [x] There is one unseal key (1 share, threshold 1), kept safe outside
+  this cluster (2026-10-25); without it the data in Vault is lost
 - [x] Unsealing is in the SSD install steps
 - [ ] After a week without problems: delete the three snapshots
 
@@ -181,7 +197,7 @@ Findings from 2026-10-25:
 - Velero copies the CNPG and MariaDB data directories file by file while
   the databases run, which is not a consistent copy. The Longhorn backups
   start from a snapshot and are crash-consistent. CNPG can back up to
-  MinIO itself (barman cloud plugin), with point-in-time recovery.
+  Garage itself (barman cloud plugin), with point-in-time recovery.
 - The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
   maintenance job pods) was fixed that afternoon by
   `velero/skip-emptydir-policy.yaml`.
@@ -189,10 +205,31 @@ Findings from 2026-10-25:
 - [ ] Whether the immich library stays in Velero. It stays for now
   (2026-10-25); decide again with the off-site copy (2)
 - [ ] CNPG backups through the barman cloud plugin
-- [ ] Velero's target, MinIO on the NAS, is no longer maintained: no
+- [x] Velero's target, MinIO on the NAS, is no longer maintained: no
   community images since October 2025, maintenance mode since December
-  2025, and the repository is archived. Move Velero to the Garage from 2
-  and switch MinIO off
+  2025, and the repository is archived. Since 2026-10-26 Velero backs up
+  to Garage on the same NAS (`nas/garage`, PR #35). The bucket was copied
+  with rclone (185 GiB in about 6 hours, then a catch-up sync just before
+  the switch), so the existing backups and kopia repositories carried on:
+  after the switch all 55 backups were still listed, and a manual backup
+  of ntfy found the existing repository and used the night's snapshot as
+  its parent. Two things made the NAS slow during the copy: Garage on
+  LMDB with `metadata_fsync`, ~270 disk writes/s while idle (now SQLite,
+  see `nas/garage/garage.toml`), and MinIO's scanner, which kept one disk
+  at 97% busy. The scanner is now at `slowest` (`mc admin config set local
+  scanner speed=slowest` in the MinIO container); it may have slowed
+  Velero's backups to MinIO as well
+- [ ] Switch MinIO off once Velero has run on Garage for a week (from
+  2026-11-02): stop the container, then delete `/volume1/minio` (343 GiB)
+  and the hand-made Secret `velero/cloud-credentials` (MinIO's key). Only
+  MinIO still has the kopia repositories of namespaces that are gone
+  (~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
+  mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
+  whether any of them is worth keeping
+- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
+  2026-10-26 next to a 270 MB database, and going by the folders in
+  `meta/snapshots`, each 6-hourly metadata snapshot takes about 70
+  minutes. Not growing; check again after a week of normal use
 
 ### 6. Monitoring
 
@@ -265,8 +302,42 @@ Counted 2026-10-25 over the 73 running containers:
   external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
   default) still have none; `default` can reach the LAN and everything
   without a policy
-- [ ] Requests and memory limits, the big ones first (immich, authentik,
-  Argo CD's repo-server)
+- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
+  had none except the Vault server, from `kubectl top` every 5 minutes
+  from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
+  floor). Rolled out in about eight minutes; both CNPG instances
+  restarted and came back healthy, the OIDC endpoints, the logins and
+  immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
+  1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
+  requested. Still without requests: Longhorn (22 containers, its chart),
+  the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
+  kube-system
+- [ ] Memory limits, once there are daytime numbers: the night ones miss
+  immich's machine learning at work (it idles at 234Mi)
+- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
+  2026-10-25, its third restart: a crawler asked for 88 pages and assets
+  within a few seconds (normally ~6 a minute), Apache started a PHP worker
+  for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
+  the container went past its 1Gi limit. Unrelated to the
+  NetworkPolicies. Now capped at 12 workers through the chart's
+  `apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
+  with `apache2ctl -t` in the same image first. Replaying the burst (90
+  page requests, 40 at a time): all 200, at most 13 Apache processes and
+  572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
+  peak) instead of a crash
+- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
+  cluster between Envoy and WordPress that keeps anonymous GETs for a few
+  minutes and passes anything with a WordPress cookie through. A page
+  renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
+  waiting in a burst is queueing; the cap alone already prevents the crash
+- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
+  `litespeed-cache` only caches pages on a LiteSpeed server, so it does
+  little here, `wordpress-importer` is usually for a one-off import, and
+  `code-snippets` runs PHP from the dashboard
+- [ ] Rate limiting per visitor in Envoy needs the client's address, which
+  node001 loses: it passes TLS through, so every request arrives from
+  46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
+  (ClientTrafficPolicy `enableProxyProtocol`)
 - [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
   mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
   (2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
@@ -277,8 +348,10 @@ Counted 2026-10-25 over the 73 running containers:
 
 ### 8. Renovate and CI
 
-10 Renovate PRs are open, the oldest from February, and nothing checks a
-PR before Argo CD applies it.
+On 2026-10-25 10 Renovate PRs were open, the oldest from February, and
+nothing checked a PR before Argo CD applied it. All of them have since been
+merged or closed (below), none are open (checked 2026-10-25), and CI checks
+every PR.
 
 - [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
   every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
@@ -314,34 +387,30 @@ PR before Argo CD applies it.
   cover every app folder, a regex manager reads `imageName` in
   `*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
   (those need pg_upgrade or a dump). Expect a batch of new PRs
-  (2026-10-25; `renovate-config-validator` passes)
-- [ ] The open PRs, all validated on 2026-10-25 against main as it is now
-  (rebases requested, so the checks show on the PRs once Renovate has
-  pushed). Merging is yours; merging deploys:
-  - #28 cert-manager v1.21.3: security release, "All users should
-    upgrade" (drops the compromised eMudhra root, fixes an ARI renewal
-    loop and a private key copy path). Merge first
-  - #24 external-snapshotter v8.2.1 → v8.7.0: 8.6 moved VolumeGroupSnapshot
-    to v1 and 8.7 turns its feature gate on; the cluster already serves the
-    v1 CRDs. Merge; close #15
-  - #25 WordPress chart 0.15.0 → 0.16.7: the image is pinned
-    (7.1.3-php8.3-apache), so the render only changes labels and
-    checksums (a pod restart). Merge; #19 (0.15.1) is superseded
-  - #26 Gitea 1.27.3 → 28.2.0 (Gitea renumbered after 1.27): security
-    fixes; the breaking changes are to egress host lists
-    (`ALLOWED_DOMAINS` and friends), none of which are set here. Database
-    migrations can't be undone, so merge after a fresh backup
-  - #23 ntfy v2.28.0 → v2.34.0: six releases in 17 days; 2.31.0 had a
-    database load bug that ntfy.sh rolled back (fixed in 2.31.1). No
-    breaking migrations. Fine, or wait a few days for 2.34 to settle
-  - #17 external-secrets 2.13.0 → 2.14.0: no breaking changes in the
-    notes; every secret goes through it, so merge when there's time to
-    watch a force-sync
-  - #27 go-dqlite v3.0.4 for `tools/dqlite-vacuum`: builds; harmless
-  - #29 yaml.v3 for the same tool: builds, but the code imports yaml.v2,
-    so it only adds an unused requirement. Close
-  - #31 python 3.15.1 for `gatus/backup-check.yaml`: the check's tests
-    (made-up data for every failure) pass on that image. Merge
+  (2026-10-25). They only come once fewer than 10 Renovate PRs are open
+  (`config:recommended`'s `prConcurrentLimit`); until then they wait
+  under "Rate-Limited" on the Dependency Dashboard (#2)
+  - The other folders were left out with a `(?!…)` lookahead. Renovate
+    compiles these patterns with RE2, which has no lookahead, so it
+    rejected the config and stopped opening PRs (#32). Each manager now
+    takes every folder and skips the others through its own
+    `ignorePaths`. Renovate 44.181.6 with RE2 loaded fails the old config
+    with #32's error and passes the new one, and its matcher picks the
+    same 166 files as the old pattern did
+- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
+  v1.21.3 (security release; all three pods on v1.21.3), #24
+  external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
+  v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
+  0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
+  ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
+  day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
+  28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
+  its own beforehand, the nightly ones are the way back)
+- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
+  manual rebase (#33 had added values on the next line, which git counts
+  as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
+  ExternalSecrets refreshed on a force-sync; the 25th is the generated
+  `gatus/backup-check-token`, which never refreshes by design
 
 ### 9. One node
 
@@ -357,6 +426,12 @@ step after that, if downtime starts to matter.
   `backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
   completed kopia maintenance jobs of repositories that no longer exist
   (nextcloud, owntracks, sogo, sogo-mail, ttrss)
+- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
+  volumes over the file paths `data/config.custom.php` and
+  `data/config-user.custom.php`, so FreshRSS sees two empty directories
+  there and the volumes hold nothing (0 bytes in every backup). Drop both
+  mounts and the two PVCs, or make them ConfigMap files if custom config
+  is wanted
 - [ ] `kubectl delete namespace secrets`, empty since its app went
 - [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
   the weeklies since 08-02. Velero doesn't restore from a Failed backup;
@@ -398,7 +473,7 @@ critical / high:
 | tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
 | longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
 | actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
-| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
+| gitea/gitea:1.27.3 | 8 | 43 | now 28.2.0 (PR #26, 2026-10-25); not scanned again |
 | immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
 | towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
 | rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |

docs/upgrade-k8s-1.33.md

@@ -282,6 +282,13 @@ whatever time of day the hourly schedule had drifted to.
     migrations): 19-33 ms per write, single 2s windows near 2 s per write,
     pressure 78%. The CSI sidecars and Longhorn's global manager lost
     their leases at 01:31
+  Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
+  11:30, leaving out the CNPG range cancels): all 67 minutes with errors
+  had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
+  "full" pressure at 45-78%, against a median minute peak of 24 ms. The
+  worst minute after the merges was the rollout of eight merged PRs at
+  2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
+  CNPG and Envoy Gateway held their 60s leases through all of it
   Ubuntu's sysstat already keeps per-device history every 10 minutes
   without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
   ~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging

gateway/proxy-protocol.yaml

@@ -0,0 +1,23 @@
+# node001 passes TLS for the tunneled hosts through without opening it, so
+# without this every request arrives from 46.224.26.65. Wordfence locks that
+# address out after 20 failed logins (2026-10-26), and with it everyone who
+# logs in to scoutingmbg.nl. node001 puts a PROXY header with the visitor's
+# address in front of each connection; Envoy takes the address from there
+# and adds it to X-Forwarded-For.
+#
+# Optional, because s2 and ntfy2 point straight at home and their clients
+# send no header, and neither does node001's proxy on port 80. The price:
+# whoever connects to the home address directly can send a header of their
+# own and claim any address.
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: ClientTrafficPolicy
+metadata:
+  name: proxy-protocol
+  namespace: envoy-gateway-system
+spec:
+  targetRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: homelab
+  proxyProtocol:
+    optional: true

gatus/backup-check.yaml

@@ -256,7 +256,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: check
-              image: python:3.13.16-alpine3.24
+              image: python:3.15.1-alpine3.24
               command: ["python", "/script/check.py"]
               env:
                 - name: PYTHONDONTWRITEBYTECODE

gitea/gitea-deployment.yaml

@@ -20,7 +20,7 @@ spec:
         fsGroup: 1000
       containers:
         - name: gitea
-          image: gitea/gitea:1.27.3
+          image: gitea/gitea:28.2.0
           env:
             - name: USER_UID
               value: "1000"

immich/helm-immich.yaml

@@ -127,6 +127,9 @@ spec:
                       TRANSFORMERS_CACHE: /cache
                       HF_XET_CACHE: /cache/huggingface-xet
                       MPLCONFIGDIR: /cache/matplotlib-config
+                      # The optimized models of v3.3.0: faster, less memory,
+                      # and the same results, so no jobs to run again.
+                      MACHINE_LEARNING_MODEL_REVISION: v2
                       # Trust store without the compromised eMudhra
                       # intermediates; requests would otherwise use certifi.
                       SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
@@ -139,11 +142,19 @@ spec:
                 globalMounts:
                   - path: /etc/ssl/certs
                     readOnly: true
+              # The models and their one-off preparation, kept across restarts.
+              # Downloaded again if lost, so no backups: the no-backup group
+              # keeps it out of Longhorn's default one (backup-daily), and
+              # Velero skips that label too (velero/skip-emptydir-policy.yaml).
               cache:
                 enabled: true
-                # Optional: to avoid downloading the ML models every start, use
-                # a persistentVolumeClaim with size, accessMode and storageClass
-                type: emptyDir
+                type: persistentVolumeClaim
+                storageClass: longhorn
+                accessMode: ReadWriteOnce
+                size: 10Gi
+                labels:
+                  recurring-job.longhorn.io/source: enabled
+                  recurring-job-group.longhorn.io/no-backup: enabled
 
   destination:
     server: https://kubernetes.default.svc

nas/garage/README.md

@@ -0,0 +1,35 @@
+# Garage on the NAS
+
+S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
+the cluster. Replaced MinIO, which is no longer maintained. One node, data
+in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
+3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
+
+The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
+in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
+its PATH.
+
+## Setup
+
+`garage.toml` here has placeholders for the two secrets; on the NAS they
+were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
+`openssl rand -base64 32` (`admin_token`), and only exist there.
+
+```
+cd /volume1/docker/garage
+docker compose up -d
+g="docker exec garage /garage"
+$g layout assign -z nas -c 800G <node id from $g status>
+$g layout apply --version 1
+$g bucket create velero-backups
+$g key create velero
+$g bucket allow --read --write --key velero velero-backups
+```
+
+Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
+`secret_access_key`) and reaches Velero through
+`velero/external-secret.yaml`. Without the cluster, `$g key info velero
+--show-secret` on the NAS shows it again.
+
+Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
+after merging, `docker compose pull && docker compose up -d` on the NAS.

nas/garage/compose.yaml

@@ -0,0 +1,12 @@
+services:
+  garage:
+    image: dxflrs/garage:v2.5.0
+    container_name: garage
+    restart: unless-stopped
+    ports:
+      - "3900:3900"  # S3 API
+      - "3903:3903"  # admin API
+    volumes:
+      - ./garage.toml:/etc/garage.toml:ro
+      - ./meta:/var/lib/garage/meta
+      - ./data:/var/lib/garage/data

nas/garage/garage.toml

@@ -0,0 +1,29 @@
+# One node on the Synology. Velero's kopia data is encrypted, so
+# compression would only cost CPU.
+#
+# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
+# shutdown, and a single node has no other node to repair from. With
+# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
+# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
+# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
+# with `garage convert-db`.
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+db_engine = "sqlite"
+metadata_fsync = true
+data_fsync = true
+metadata_auto_snapshot_interval = "6h"
+replication_factor = 1
+compression_level = "none"
+
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "127.0.0.1:3901"
+rpc_secret = "@RPC_SECRET@"
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "[::]:3900"
+
+[admin]
+api_bind_addr = "[::]:3903"
+admin_token = "@ADMIN_TOKEN@"

nix-infra-machine/nodes/node001.nix

@@ -173,12 +173,18 @@ in
     #   - mail.rubenhensen.nl (and anything else) → local stalwart on 8443
     #   - tunneled hosts → home cluster ingress on 443
     #   - s1 / ntfy1.rubenhensen.nl → local nginx on 8444 (Gatus, ntfy)
+    #
+    # Each connection starts with a PROXY header that carries the visitor's
+    # address, so the home cluster sees who is asking instead of this box
+    # (gateway/proxy-protocol.yaml in k8scd). nginx sends the header to
+    # every upstream or none, so the local ones go through a hop on
+    # 8445/8446 that takes it off again.
     streamConfig = ''
       map $ssl_preread_server_name $tunnel_upstream {
 ${sniMapEntries}
-        s1.rubenhensen.nl 127.0.0.1:8444;
-        ntfy1.rubenhensen.nl 127.0.0.1:8444;
-        default 127.0.0.1:8443;
+        s1.rubenhensen.nl 127.0.0.1:8446;
+        ntfy1.rubenhensen.nl 127.0.0.1:8446;
+        default 127.0.0.1:8445;
       }
 
       server {
@@ -186,6 +192,17 @@ ${sniMapEntries}
         listen [::]:443;
         proxy_pass $tunnel_upstream;
         ssl_preread on;
+        proxy_protocol on;
+      }
+
+      server {
+        listen 127.0.0.1:8445 proxy_protocol;
+        proxy_pass 127.0.0.1:8443;
+      }
+
+      server {
+        listen 127.0.0.1:8446 proxy_protocol;
+        proxy_pass 127.0.0.1:8444;
       }
     '';
 

ntfy/ntfy-deployment.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: ntfy
-          image: binwiederhier/ntfy:v2.28.0
+          image: binwiederhier/ntfy:v2.34.0
           args: ["serve"]
           env:
             - name: NTFY_BASE_URL

renovate.json

@@ -4,9 +4,18 @@
     "config:recommended",
     "customManagers:githubActionsVersions"
   ],
+  "timezone": "Europe/Amsterdam",
   "argocd": {
+    "description": "Every app folder; Renovate's regexes are RE2, which has no lookahead, so the other folders are skipped through ignorePaths",
     "managerFilePatterns": [
-      "/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
+      "/^[^/]+/.+\\.yaml$/"
+    ],
+    "ignorePaths": [
+      "nix-infra-machine/**",
+      "backup-testing/**",
+      "docs/**",
+      "tools/**",
+      ".github/**"
     ]
   },
   "helm-values": {
@@ -15,8 +24,16 @@
     ]
   },
   "kubernetes": {
+    "description": "Every app folder, as for argocd",
     "managerFilePatterns": [
-      "/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
+      "/^[^/]+/.+\\.yaml$/"
+    ],
+    "ignorePaths": [
+      "nix-infra-machine/**",
+      "backup-testing/**",
+      "docs/**",
+      "tools/**",
+      ".github/**"
     ]
   },
   "customManagers": [
@@ -72,6 +89,27 @@
         "major"
       ],
       "enabled": false
+    },
+    {
+      "description": "Patch updates of apps that hold little state and are easy to roll back, and of CI's tools, merge themselves once CI passes: three days after the release, in the evening or at the weekend, when the cluster can be reached. Not the databases, operators, Vault, authentik, Longhorn, Velero, immich, Gitea or WordPress. Renovate merges itself; the repo has no branch protection for GitHub's auto-merge to wait on",
+      "matchFileNames": [
+        "ntfy/**",
+        "gatus/**",
+        "freshrss/**",
+        "serpbear/**",
+        "actual-budget/**",
+        ".github/workflows/**"
+      ],
+      "matchUpdateTypes": [
+        "patch"
+      ],
+      "minimumReleaseAge": "3 days",
+      "automerge": true,
+      "platformAutomerge": false,
+      "automergeSchedule": [
+        "* 17-21 * * 1-5",
+        "* 10-21 * * 0,6"
+      ]
     }
   ]
 }

tools/dqlite-vacuum/go.mod

@@ -3,7 +3,7 @@ module dqlite-vacuum
 go 1.27.3
 
 require (
-	github.com/canonical/go-dqlite/v3 v3.0.1
+	github.com/canonical/go-dqlite/v3 v3.0.4
 	gopkg.in/yaml.v2 v2.4.0
 )
 

tools/dqlite-vacuum/go.sum

@@ -2,7 +2,10 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
 github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
 github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
 github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
+github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
+github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -26,7 +29,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

velero/external-secret.yaml

@@ -0,0 +1,30 @@
+# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
+# AWS credentials file format the aws plugin and kopia read.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: velero-s3-credentials
+spec:
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  refreshInterval: 15m
+  target:
+    name: velero-s3-credentials
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        cloud: |
+          [default]
+          aws_access_key_id={{ .access_key_id }}
+          aws_secret_access_key={{ .secret_access_key }}
+  data:
+    - secretKey: access_key_id
+      remoteRef:
+        key: kv/velero-s3
+        property: access_key_id
+    - secretKey: secret_access_key
+      remoteRef:
+        key: kv/velero-s3
+        property: secret_access_key

velero/nas-svc.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: nas-minio-external
-spec:
-  type: ExternalName
-  externalName: 192.168.1.99
-  ports:
-    - port: 9000
-      protocol: TCP

velero/skip-emptydir-policy.yaml

@@ -1,6 +1,8 @@
 # Volume policy for the backup schedules. emptyDir volumes hold nothing
 # worth keeping, and backing them up fails when the pod is gone before its
 # volume is read, as happens with Velero's own repository maintenance jobs.
+# PVCs in Longhorn's no-backup group (caches, such as immich's ML models)
+# are skipped too.
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -14,3 +16,8 @@ data:
         - emptyDir
       action:
         type: skip
+    - conditions:
+        pvcLabels:
+          recurring-job-group.longhorn.io/no-backup: enabled
+      action:
+        type: skip

velero/velero-helm.yaml

@@ -27,20 +27,22 @@ spec:
               - mountPath: /target
                 name: plugins
         configuration:
+          # Garage on the NAS (nas/garage). Until 2026-10-26 this was MinIO on
+          # the same NAS; the bucket was copied over with rclone, so the
+          # existing backups and kopia repositories carried on.
           backupStorageLocation:
             - name: default
               bucket: velero-backups
               provider: aws
               config:
-                region: minio
+                region: garage
                 s3ForcePathStyle: true
-                s3Url: http://192.168.1.99:9000
-                insecureSkipTLSVerify: "true"
+                s3Url: http://192.168.1.99:3900
           volumeSnapshotLocation:
             - name: default
               provider: aws
               config:
-                region: minio
+                region: garage
           features: EnableCSI
           # Kopia maintenance ran every hour for each of the 15 repositories,
           # and once a day as a full run of several minutes that writes its
@@ -59,11 +61,8 @@ spec:
                   memoryRequest: 256Mi
                   memoryLimit: 2Gi
         credentials:
-          # Use existing secret created by External Secrets Operator
-          # existingSecret: velero-minio-credentials
-          # extraEnvVars:
-          # AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
-          existingSecret: cloud-credentials
+          # From Vault, velero/external-secret.yaml
+          existingSecret: velero-s3-credentials
         deployNodeAgent: true
         nodeAgent:
           containerSecurityContext: