backup-testing: read Velero's key for Garage

chore(deps): update helm chart external-secrets to v2.14.0

apps/templates/cert-manager-helm.yaml

@@ -12,7 +12,7 @@ spec:
   project: default
   source:
     repoURL: https://charts.jetstack.io
-    targetRevision: v1.21.2
+    targetRevision: v1.21.3
     chart: cert-manager
     helm:
       parameters:

apps/templates/external-secrets-helm.yaml

@@ -17,7 +17,7 @@ spec:
   sources:
     - chart: external-secrets
       repoURL: https://charts.external-secrets.io
-      targetRevision: 2.13.0
+      targetRevision: 2.14.0
       helm:
         values: |
           resources:

apps/templates/mbgwp-official-helm.yaml

@@ -12,7 +12,7 @@ spec:
   project: default
   source:
     repoURL: https://groundhog2k.github.io/helm-charts/
-    targetRevision: 0.15.0
+    targetRevision: 0.16.7
     chart: wordpress
     helm:
       values: |
@@ -101,6 +101,23 @@ spec:
           timeoutSeconds: 5
           failureThreshold: 30
 
+        # apache2.conf includes ports.conf after mods-enabled/*.conf, so the
+        # prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
+        # 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
+        # crawler asking for 88 pages at once got the container OOMKilled on
+        # 2026-10-25. With 12 workers a burst waits in the listen backlog
+        # instead. Listen is the chart's default (containerPort).
+        apachePortsConfig: |
+          Listen 8000
+          <IfModule mpm_prefork_module>
+            StartServers            4
+            MinSpareServers         2
+            MaxSpareServers         6
+            ServerLimit             12
+            MaxRequestWorkers       12
+            MaxConnectionsPerChild  1000
+          </IfModule>
+
         # Custom PHP configuration
         customPhpConfig: |
           upload_max_filesize = 64M

apps/templates/snapshot-controller-helm.yaml

@@ -7,7 +7,7 @@ spec:
   project: default
   source:
     repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
-    targetRevision: v8.2.1
+    targetRevision: v8.7.0
     path: deploy/kubernetes/snapshot-controller
     kustomize:
       patches:

backup-testing/restore-test.sh

@@ -3,7 +3,7 @@
 # k3s in Docker, to prove the backups can be restored. Nothing on the NAS
 # changes:
 # - one backup (backups/<name>/) and the kopia repositories of the chosen
-#   namespaces (kopia/<namespace>/) are copied read-only from MinIO with
+#   namespaces (kopia/<namespace>/) are copied read-only from the NAS with
 #   rclone into a Docker volume. A second Velero on the real bucket could
 #   run kopia maintenance on the same repositories as the cluster's own
 # - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
@@ -19,7 +19,7 @@
 # Defaults: the newest Completed daily-backup, and ntfy freshrss
 # actual-budget. Not gatus: a restored Gatus would start sending alerts.
 #
-# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
+# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
 # into a temporary directory, and the copied backup holds every Secret of
 # the cluster; all of it is deleted at the end. KEEP=1 keeps the test
 # cluster (kubeconfig in the printed directory) until you run it with
@@ -33,8 +33,6 @@
 set -euo pipefail
 
 PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
-MINIO=${MINIO:-http://192.168.1.99:9000}
-BUCKET=velero-backups
 PORT=${PORT:-16443}
 K3S=rancher/k3s:v1.34.9-k3s1
 RCLONE=rclone/rclone:1.75.1
@@ -73,6 +71,18 @@ wait_for() {
   done
 }
 
+# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
+# request signed for the wrong region (rclone defaults to us-east-1); MinIO
+# did so with a bare 400 on HEAD.
+if [[ -n ${FROM_VOLUME:-} ]]; then
+  BUCKET=${BUCKET:-velero-backups}
+else
+  bsl=$(pk -n velero get backupstoragelocations default -o json)
+  S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
+  REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
+  BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
+fi
+
 backup=${1:-}
 shift || true
 namespaces=("$@")
@@ -91,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
   docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
 else
 echo "== copy from the NAS (read-only)"
-(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
+(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
   awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
                  /aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
-  printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env")
+  printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
 copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
 copy "backups/$backup"
 for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done

docs/immich-scouts-online.md

@@ -123,15 +123,13 @@ D. auth.scoutingmbg.nl (2026-10-25)
 
 ## Open points
 
-- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
-  foto.scoutingmbg.nl from a CNAME to `mbg.hensen.io` (the home IP) to a
-  CNAME to `rubenhensen.nl`, which is node001 on Hetzner. Only ns2 has it
-  (serial 2026102471); ns1 is stuck on serial 2026060640 from June and
-  still answers `mbg.hensen.io`. node001 now tunnels foto.scoutingmbg.nl to
-  the home cluster like its other hosts, so both answers work. Keep
-  hensen.io until ns1 has caught up; ask the DNS host to fix ns1, and
-  decide whether foto should stay behind node001 or point straight home
-  (CNAME `scoutingmbg.nl`)
+- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
+  scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
+  all the records it moved, and node001 tunnels all four home. ns1 lagged
+  behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
+  uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
+  `a` now means node001, which is fine: the WordPress site sends no mail
+  and the group's mail goes through Google (`mx`)
 - Public shared links become public for real once sol-auth is gone.
 - Photos leiding back up from their phones stay private until they add
   them to a shared album.

docs/longhorn-second-ssd.md

@@ -38,14 +38,15 @@ between them live.
 
 ## Before: shrink what has to move
 
-- [ ] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
+- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
   snapshots from before October 2026, nearly all on immich-claim
   (pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
   2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
   longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
   don't depend on them. Deleting a snapshot merges it into the next one,
   which writes a lot, so do this in a quiet hour, one volume at a time.
-  Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
+  Happened on 2026-10-24, from a Claude Code session at Ruben's request
+  (the 45 snapshots from before October 2026, small volumes first): immich-claim's
   old snapshots were removed at 14:43 UTC and Longhorn merged them one
   after another, about 45 minutes each, until 23:15. For those 8.5 hours
   sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on

docs/roadmap.md

@@ -13,7 +13,8 @@ volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
 NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
 01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
 and s2 (home) watch each other and alert through ntfy; s2 also gets an
-hourly report on the backups and Vault's seal from inside the cluster.
+hourly report from inside the cluster on the backups, Vault's seal, the
+Argo CD apps and the CNPG clusters.
 Every app namespace has NetworkPolicies, and CI renders and validates all
 apps on every PR. node001 runs mail and tunnels the public hosts home.
 
@@ -33,8 +34,9 @@ docs/longhorn-second-ssd.md:
 - [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
   (`backend-*.tar`, `dqlite-pre-*`)
 - [ ] Log in to immich as a jeugdlid (quota 0)
-- [ ] Back to 1.36/stable once it is at v1.36.5 or later. The snap is held,
-  so microk8s gets no updates at all until someone refreshes it
+- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
+  2026-10-25). The snap is held, so microk8s gets no updates at all until
+  someone refreshes it
 - [ ] Second SSD, docs/longhorn-second-ssd.md
 - [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
 
@@ -48,8 +50,9 @@ group's ns2: ns1 is stuck on an old serial and still answers
 mbg.hensen.io (docs/immich-scouts-online.md).
 
 - [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
-- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
-  rubenhensen.nl too
+- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
+  `rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
+  cluster uses hensen.io anymore. hensen.io can expire
 
 ### 2. Off-site backups
 
@@ -97,8 +100,17 @@ friend.
   ReadOnly location: data restored byte for byte (same sha256), deployment
   available, everything removed afterwards. Two pitfalls found on the way
   are in the script's header. Not yet run against the real backups:
-- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from
-  the cluster), then schedule it
+- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
+  daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
+  volume came back with its backed-up byte count (actual-budget 8.4 MB,
+  freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
+  first run failed on a 400 from MinIO: rclone signed for us-east-1 while
+  MinIO runs region `minio`; the script now reads endpoint, region and
+  bucket from Velero's location. Copying freshrss's kopia repository took
+  14 of the ~17 minutes
+- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
+  the cluster itself: the Mac (launchd) or node001, ideally with a
+  read-only MinIO user
 
 ### 3. authentik 2026.2.1 → 2026.8.5
 
@@ -153,8 +165,8 @@ unseals it.
   - The OIDC job called `vault auth tune -listing_visibility`, a flag that
     doesn't exist, so the OIDC mount's listing visibility was never set.
     Fixed; the job now uses the 2.2.1 CLI instead of 1.15
-- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
-  places outside this cluster; without it the data in Vault is lost
+- [x] There is one unseal key (1 share, threshold 1), kept safe outside
+  this cluster (2026-10-25); without it the data in Vault is lost
 - [x] Unsealing is in the SSD install steps
 - [ ] After a week without problems: delete the three snapshots
 
@@ -265,8 +277,42 @@ Counted 2026-10-25 over the 73 running containers:
   external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
   default) still have none; `default` can reach the LAN and everything
   without a policy
-- [ ] Requests and memory limits, the big ones first (immich, authentik,
-  Argo CD's repo-server)
+- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
+  had none except the Vault server, from `kubectl top` every 5 minutes
+  from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
+  floor). Rolled out in about eight minutes; both CNPG instances
+  restarted and came back healthy, the OIDC endpoints, the logins and
+  immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
+  1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
+  requested. Still without requests: Longhorn (22 containers, its chart),
+  the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
+  kube-system
+- [ ] Memory limits, once there are daytime numbers: the night ones miss
+  immich's machine learning at work (it idles at 234Mi)
+- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
+  2026-10-25, its third restart: a crawler asked for 88 pages and assets
+  within a few seconds (normally ~6 a minute), Apache started a PHP worker
+  for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
+  the container went past its 1Gi limit. Unrelated to the
+  NetworkPolicies. Now capped at 12 workers through the chart's
+  `apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
+  with `apache2ctl -t` in the same image first. Replaying the burst (90
+  page requests, 40 at a time): all 200, at most 13 Apache processes and
+  572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
+  peak) instead of a crash
+- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
+  cluster between Envoy and WordPress that keeps anonymous GETs for a few
+  minutes and passes anything with a WordPress cookie through. A page
+  renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
+  waiting in a burst is queueing; the cap alone already prevents the crash
+- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
+  `litespeed-cache` only caches pages on a LiteSpeed server, so it does
+  little here, `wordpress-importer` is usually for a one-off import, and
+  `code-snippets` runs PHP from the dashboard
+- [ ] Rate limiting per visitor in Envoy needs the client's address, which
+  node001 loses: it passes TLS through, so every request arrives from
+  46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
+  (ClientTrafficPolicy `enableProxyProtocol`)
 - [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
   mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
   (2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
@@ -314,34 +360,24 @@ PR before Argo CD applies it.
   cover every app folder, a regex manager reads `imageName` in
   `*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
   (those need pg_upgrade or a dump). Expect a batch of new PRs
-  (2026-10-25; `renovate-config-validator` passes)
-- [ ] The open PRs, all validated on 2026-10-25 against main as it is now
-  (rebases requested, so the checks show on the PRs once Renovate has
-  pushed). Merging is yours; merging deploys:
-  - #28 cert-manager v1.21.3: security release, "All users should
-    upgrade" (drops the compromised eMudhra root, fixes an ARI renewal
-    loop and a private key copy path). Merge first
-  - #24 external-snapshotter v8.2.1 → v8.7.0: 8.6 moved VolumeGroupSnapshot
-    to v1 and 8.7 turns its feature gate on; the cluster already serves the
-    v1 CRDs. Merge; close #15
-  - #25 WordPress chart 0.15.0 → 0.16.7: the image is pinned
-    (7.1.3-php8.3-apache), so the render only changes labels and
-    checksums (a pod restart). Merge; #19 (0.15.1) is superseded
-  - #26 Gitea 1.27.3 → 28.2.0 (Gitea renumbered after 1.27): security
-    fixes; the breaking changes are to egress host lists
-    (`ALLOWED_DOMAINS` and friends), none of which are set here. Database
-    migrations can't be undone, so merge after a fresh backup
-  - #23 ntfy v2.28.0 → v2.34.0: six releases in 17 days; 2.31.0 had a
-    database load bug that ntfy.sh rolled back (fixed in 2.31.1). No
-    breaking migrations. Fine, or wait a few days for 2.34 to settle
-  - #17 external-secrets 2.13.0 → 2.14.0: no breaking changes in the
-    notes; every secret goes through it, so merge when there's time to
-    watch a force-sync
-  - #27 go-dqlite v3.0.4 for `tools/dqlite-vacuum`: builds; harmless
-  - #29 yaml.v3 for the same tool: builds, but the code imports yaml.v2,
-    so it only adds an unused requirement. Close
-  - #31 python 3.15.1 for `gatus/backup-check.yaml`: the check's tests
-    (made-up data for every failure) pass on that image. Merge
+  (2026-10-25; `renovate-config-validator` passes). They only come once
+  fewer than 10 Renovate PRs are open (`config:recommended`'s
+  `prConcurrentLimit`); until then they wait under "Rate-Limited" on the
+  Dependency Dashboard (#2)
+- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
+  v1.21.3 (security release; all three pods on v1.21.3), #24
+  external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
+  v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
+  0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
+  ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
+  day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
+  28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
+  its own beforehand, the nightly ones are the way back)
+- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
+  manual rebase (#33 had added values on the next line, which git counts
+  as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
+  ExternalSecrets refreshed on a force-sync; the 25th is the generated
+  `gatus/backup-check-token`, which never refreshes by design
 
 ### 9. One node
 
@@ -357,6 +393,12 @@ step after that, if downtime starts to matter.
   `backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
   completed kopia maintenance jobs of repositories that no longer exist
   (nextcloud, owntracks, sogo, sogo-mail, ttrss)
+- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
+  volumes over the file paths `data/config.custom.php` and
+  `data/config-user.custom.php`, so FreshRSS sees two empty directories
+  there and the volumes hold nothing (0 bytes in every backup). Drop both
+  mounts and the two PVCs, or make them ConfigMap files if custom config
+  is wanted
 - [ ] `kubectl delete namespace secrets`, empty since its app went
 - [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
   the weeklies since 08-02. Velero doesn't restore from a Failed backup;

docs/upgrade-k8s-1.33.md

@@ -282,6 +282,13 @@ whatever time of day the hourly schedule had drifted to.
     migrations): 19-33 ms per write, single 2s windows near 2 s per write,
     pressure 78%. The CSI sidecars and Longhorn's global manager lost
     their leases at 01:31
+  Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
+  11:30, leaving out the CNPG range cancels): all 67 minutes with errors
+  had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
+  "full" pressure at 45-78%, against a median minute peak of 24 ms. The
+  worst minute after the merges was the rollout of eight merged PRs at
+  2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
+  CNPG and Envoy Gateway held their 60s leases through all of it
   Ubuntu's sysstat already keeps per-device history every 10 minutes
   without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
   ~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging

gatus/backup-check.yaml

@@ -256,7 +256,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: check
-              image: python:3.13.16-alpine3.24
+              image: python:3.15.1-alpine3.24
               command: ["python", "/script/check.py"]
               env:
                 - name: PYTHONDONTWRITEBYTECODE

gitea/gitea-deployment.yaml

@@ -20,7 +20,7 @@ spec:
         fsGroup: 1000
       containers:
         - name: gitea
-          image: gitea/gitea:1.27.3
+          image: gitea/gitea:28.2.0
           env:
             - name: USER_UID
               value: "1000"

nas/garage/README.md

@@ -0,0 +1,35 @@
+# Garage on the NAS
+
+S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
+the cluster. Replaced MinIO, which is no longer maintained. One node, data
+in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
+3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
+
+The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
+in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
+its PATH.
+
+## Setup
+
+`garage.toml` here has placeholders for the two secrets; on the NAS they
+were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
+`openssl rand -base64 32` (`admin_token`), and only exist there.
+
+```
+cd /volume1/docker/garage
+docker compose up -d
+g="docker exec garage /garage"
+$g layout assign -z nas -c 800G <node id from $g status>
+$g layout apply --version 1
+$g bucket create velero-backups
+$g key create velero
+$g bucket allow --read --write --key velero velero-backups
+```
+
+Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
+`secret_access_key`) and reaches Velero through
+`velero/external-secret.yaml`. Without the cluster, `$g key info velero
+--show-secret` on the NAS shows it again.
+
+Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
+after merging, `docker compose pull && docker compose up -d` on the NAS.

nas/garage/compose.yaml

@@ -0,0 +1,12 @@
+services:
+  garage:
+    image: dxflrs/garage:v2.5.0
+    container_name: garage
+    restart: unless-stopped
+    ports:
+      - "3900:3900"  # S3 API
+      - "3903:3903"  # admin API
+    volumes:
+      - ./garage.toml:/etc/garage.toml:ro
+      - ./meta:/var/lib/garage/meta
+      - ./data:/var/lib/garage/data

nas/garage/garage.toml

@@ -0,0 +1,29 @@
+# One node on the Synology. Velero's kopia data is encrypted, so
+# compression would only cost CPU.
+#
+# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
+# shutdown, and a single node has no other node to repair from. With
+# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
+# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
+# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
+# with `garage convert-db`.
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+db_engine = "sqlite"
+metadata_fsync = true
+data_fsync = true
+metadata_auto_snapshot_interval = "6h"
+replication_factor = 1
+compression_level = "none"
+
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "127.0.0.1:3901"
+rpc_secret = "@RPC_SECRET@"
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "[::]:3900"
+
+[admin]
+api_bind_addr = "[::]:3903"
+admin_token = "@ADMIN_TOKEN@"

ntfy/ntfy-deployment.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: ntfy
-          image: binwiederhier/ntfy:v2.28.0
+          image: binwiederhier/ntfy:v2.34.0
           args: ["serve"]
           env:
             - name: NTFY_BASE_URL

tools/dqlite-vacuum/go.mod

@@ -3,7 +3,7 @@ module dqlite-vacuum
 go 1.27.3
 
 require (
-	github.com/canonical/go-dqlite/v3 v3.0.1
+	github.com/canonical/go-dqlite/v3 v3.0.4
 	gopkg.in/yaml.v2 v2.4.0
 )
 

tools/dqlite-vacuum/go.sum

@@ -2,7 +2,10 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
 github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
 github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
 github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
+github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
+github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -26,7 +29,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

velero/external-secret.yaml

@@ -0,0 +1,30 @@
+# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
+# AWS credentials file format the aws plugin and kopia read.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: velero-s3-credentials
+spec:
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  refreshInterval: 15m
+  target:
+    name: velero-s3-credentials
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        cloud: |
+          [default]
+          aws_access_key_id={{ .access_key_id }}
+          aws_secret_access_key={{ .secret_access_key }}
+  data:
+    - secretKey: access_key_id
+      remoteRef:
+        key: kv/velero-s3
+        property: access_key_id
+    - secretKey: secret_access_key
+      remoteRef:
+        key: kv/velero-s3
+        property: secret_access_key

velero/nas-svc.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: nas-minio-external
-spec:
-  type: ExternalName
-  externalName: 192.168.1.99
-  ports:
-    - port: 9000
-      protocol: TCP

velero/velero-helm.yaml

@@ -27,20 +27,22 @@ spec:
               - mountPath: /target
                 name: plugins
         configuration:
+          # Garage on the NAS (nas/garage). Until 2026-10-25 this was MinIO on
+          # the same NAS; the bucket was copied over with rclone, so the
+          # existing backups and kopia repositories carried on.
           backupStorageLocation:
             - name: default
               bucket: velero-backups
               provider: aws
               config:
-                region: minio
+                region: garage
                 s3ForcePathStyle: true
-                s3Url: http://192.168.1.99:9000
-                insecureSkipTLSVerify: "true"
+                s3Url: http://192.168.1.99:3900
           volumeSnapshotLocation:
             - name: default
               provider: aws
               config:
-                region: minio
+                region: garage
           features: EnableCSI
           # Kopia maintenance ran every hour for each of the 15 repositories,
           # and once a day as a full run of several minutes that writes its
@@ -59,11 +61,8 @@ spec:
                   memoryRequest: 256Mi
                   memoryLimit: 2Gi
         credentials:
-          # Use existing secret created by External Secrets Operator
-          # existingSecret: velero-minio-credentials
-          # extraEnvVars:
-          # AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
-          existingSecret: cloud-credentials
+          # From Vault, velero/external-secret.yaml
+          existingSecret: velero-s3-credentials
         deployNodeAgent: true
         nodeAgent:
           containerSecurityContext: