cnpg: back the databases up to Garage with the barman cloud plugin

Renovate: automerge patch updates of low-risk apps

actual-budget/networkpolicy.yaml

@@ -55,3 +55,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

apps/templates/plugin-barman-cloud-helm.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: plugin-barman-cloud
+  namespace: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    chart: plugin-barman-cloud
+    repoURL: https://cloudnative-pg.github.io/charts
+    targetRevision: 0.8.1
+    helm:
+      values: |
+        # CNPG's barman cloud plugin, which backs the PostgreSQL clusters
+        # up to Garage on the NAS (authentik/postgresql-backup.yaml,
+        # immich/postgresql-backup.yaml). It has to run in the operator's
+        # namespace; the chart gets its TLS certificates from cert-manager.
+        resources:
+          requests:
+            cpu: 10m
+            memory: 64Mi
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cnpg-system
+  syncPolicy:
+    syncOptions:
+    - ServerSideApply=true
+    automated:
+      prune: true
+      selfHeal: true

authentik/networkpolicy.yaml

@@ -108,7 +108,8 @@ spec:
 ---
 # The CNPG operator checks each instance on its status port and connects to
 # PostgreSQL; the instance manager inside each pod watches its Cluster
-# through the API server.
+# through the API server. The barman cloud sidecar sends the WAL and the
+# base backups to Garage on the NAS (postgresql-backup.yaml).
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -135,3 +136,34 @@ spec:
             cidr: 192.168.1.204/32
       ports:
         - port: 16443
+    - to:
+        - ipBlock:
+            cidr: 192.168.1.99/32
+      ports:
+        - port: 3900
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

authentik/postgresql-backup.yaml

@@ -0,0 +1,92 @@
+# Backups of authentik's database by CNPG itself, through the barman cloud
+# plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base backup every
+# night, and every WAL segment as soon as it is full or 5 minutes old
+# (archive_timeout). Both go to the bucket cnpg-authentik in Garage on the
+# NAS (nas/garage), so the database can be restored to any moment of the
+# last 30 days. Velero copies the data directory while PostgreSQL runs,
+# which is not a consistent copy.
+# The key is Garage's cnpg-authentik, in Vault as kv/cnpg-authentik-s3; it
+# can only reach this bucket.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: authentik-postgres-s3
+spec:
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  refreshInterval: 15m
+  target:
+    name: authentik-postgres-s3
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        ACCESS_KEY_ID: "{{ .access_key_id }}"
+        ACCESS_SECRET_KEY: "{{ .secret_access_key }}"
+        # Garage rejects requests signed for another region
+        REGION: garage
+  data:
+    - secretKey: access_key_id
+      remoteRef:
+        key: kv/cnpg-authentik-s3
+        property: access_key_id
+    - secretKey: secret_access_key
+      remoteRef:
+        key: kv/cnpg-authentik-s3
+        property: secret_access_key
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: authentik-postgres
+  annotations:
+    # The CRD comes with the plugin's own app
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  retentionPolicy: 30d
+  configuration:
+    destinationPath: s3://cnpg-authentik/
+    endpointURL: http://192.168.1.99:3900
+    s3Credentials:
+      accessKeyId:
+        name: authentik-postgres-s3
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: authentik-postgres-s3
+        key: ACCESS_SECRET_KEY
+      region:
+        name: authentik-postgres-s3
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  instanceSidecarConfiguration:
+    resources:
+      requests:
+        cpu: 10m
+        memory: 64Mi
+    env:
+      # Newer boto3 adds checksums that S3-compatible stores can reject;
+      # the plugin's docs suggest sending them only when required.
+      - name: AWS_REQUEST_CHECKSUM_CALCULATION
+        value: when_required
+      - name: AWS_RESPONSE_CHECKSUM_VALIDATION
+        value: when_required
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: authentik-postgres-daily
+spec:
+  cluster:
+    name: authentik-postgres
+  # 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right
+  # away, so WAL archiving has a base backup to start from.
+  schedule: "0 0 3 * * *"
+  immediate: true
+  backupOwnerReference: self
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

authentik/postgresql-cluster.yaml

@@ -8,6 +8,12 @@ spec:
   # start an in-place major upgrade. CNPG filled this in at creation.
   imageName: ghcr.io/cloudnative-pg/postgresql:17.4
   primaryUpdateStrategy: unsupervised
+  # WAL archiving and base backups to Garage, see postgresql-backup.yaml
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: authentik-postgres
   resources:
     requests:
       cpu: 50m

backup-testing/restore-test.sh

@@ -3,7 +3,7 @@
 # k3s in Docker, to prove the backups can be restored. Nothing on the NAS
 # changes:
 # - one backup (backups/<name>/) and the kopia repositories of the chosen
-#   namespaces (kopia/<namespace>/) are copied read-only from MinIO with
+#   namespaces (kopia/<namespace>/) are copied read-only from the NAS with
 #   rclone into a Docker volume. A second Velero on the real bucket could
 #   run kopia maintenance on the same repositories as the cluster's own
 # - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
@@ -19,7 +19,7 @@
 # Defaults: the newest Completed daily-backup, and ntfy freshrss
 # actual-budget. Not gatus: a restored Gatus would start sending alerts.
 #
-# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
+# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
 # into a temporary directory, and the copied backup holds every Secret of
 # the cluster; all of it is deleted at the end. KEEP=1 keeps the test
 # cluster (kubeconfig in the printed directory) until you run it with
@@ -71,14 +71,14 @@ wait_for() {
   done
 }
 
-# Endpoint, region and bucket as Velero itself uses them. MinIO answers a
+# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
-# request signed for the wrong region (rclone defaults to us-east-1) with a
+# request signed for the wrong region (rclone defaults to us-east-1); MinIO
-# bare 400 on HEAD.
+# did so with a bare 400 on HEAD.
 if [[ -n ${FROM_VOLUME:-} ]]; then
   BUCKET=${BUCKET:-velero-backups}
 else
   bsl=$(pk -n velero get backupstoragelocations default -o json)
-  MINIO=${MINIO:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
+  S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
   REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
   BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
 fi
@@ -101,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
   docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
 else
 echo "== copy from the NAS (read-only)"
-(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
+(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
   awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
                  /aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
-  printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$MINIO" "$REGION" >> "$WORK/nas.env")
+  printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
 copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
 copy "backups/$backup"
 for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done

bolletjestrui/networkpolicy.yaml

@@ -54,3 +54,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

docs/roadmap.md

@@ -10,11 +10,11 @@ One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
 SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
 35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
 volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
-NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
+NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to Garage
-01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
+(MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90 days).
-and s2 (home) watch each other and alert through ntfy; s2 also gets an
+Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
-hourly report from inside the cluster on the backups, Vault's seal, the
+through ntfy; s2 also gets an hourly report from inside the cluster on the
-Argo CD apps and the CNPG clusters.
+backups, Vault's seal, the Argo CD apps and the CNPG clusters.
 Every app namespace has NetworkPolicies, and CI renders and validates all
 apps on every PR. node001 runs mail and tunnels the public hosts home.
 
@@ -63,24 +63,25 @@ belong to the group.
 There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
 friend.
 
-- [ ] Ours to his bucket: Hyper Backup on the Synology, destination "S3
+- [ ] Ours to his bucket: `rclone sync` from Garage's `velero-backups`
-  Storage" with his endpoint as a custom server. Client-side encryption
+  (~190 GB) to his bucket through an rclone `crypt` remote, in Docker on
-  on: the kopia repositories use Velero's default password
+  the NAS like the copy from MinIO (see 5). Encrypted on our side: the
-  (`static-passw0rd`), so without it he could read everything, the
+  kopia repositories use Velero's default password (`static-passw0rd`),
-  group's photos included. Keep the encryption key next to the Vault
+  so without it he could read everything, the group's photos included.
-  unseal key. Source: the shared folder with MinIO's `velero-backups`
+  Keep the crypt passwords next to the Vault unseal key. Daily after the
-  bucket (~150-200 GB). Daily after the Velero backup, Smart Recycle,
+  Velero backup, ntfy on failure. Hyper Backup, the earlier plan, backs up
-  mail on failure
+  shared folders; Garage keeps its objects as blocks plus a metadata
+  database, and a copy of that folder taken while Garage runs is not
+  consistent
 - [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
   now, 500 GB to grow), and versioning or object lock on the bucket, so
   whoever takes over the NAS can't delete the off-site copy as well
-- [ ] His backups with us: Garage (not MinIO, see 5) in Container Manager
+- [ ] His backups with us: a bucket and key for him, with a quota, in the
-  on the Synology, in its own shared folder, with a bucket and key for him
+  Garage on the NAS (`nas/garage`, see 5). Only the S3 API public, as
-  and a quota. Only the S3 API public, as `s3.rubenhensen.nl` through
+  `s3.rubenhensen.nl` through node001 and Envoy with a cert-manager
-  node001 and Envoy with a cert-manager certificate; the admin API stays
+  certificate; the admin API stays on the LAN. The cluster side (DNS,
-  on the LAN. The cluster side (DNS, node001 tunnel, Certificate,
+  node001 tunnel, Certificate, HTTPRoute) goes in git. Needed: how much
-  HTTPRoute) goes in git. Needed: how much space he wants, free space on
+  space he wants, free space on the NAS
-  the NAS
 - [ ] Decide which copy of the immich library goes off-site (see 5)
 - [ ] A restore test that runs on a schedule. `backup-testing/` predates
   Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
@@ -108,9 +109,12 @@ friend.
   MinIO runs region `minio`; the script now reads endpoint, region and
   bucket from Velero's location. Copying freshrss's kopia repository took
   14 of the ~17 minutes
-- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
+- [x] Ran it again on 2026-10-26 against the copy in Garage
-  the cluster itself: the Mac (launchd) or node001, ideally with a
+  (daily-backup-20261026010028, same namespaces), before Velero switched
-  read-only MinIO user
+  over (see 5): passed, in 5 minutes instead of 17
+- [ ] Run it on a schedule. It needs Docker and an S3 key, so not in the
+  cluster itself: the Mac (launchd) or node001, ideally with a read-only
+  Garage key (`garage key create`, `garage bucket allow --read`)
 
 ### 3. authentik 2026.2.1 → 2026.8.5
 
@@ -193,7 +197,7 @@ Findings from 2026-10-25:
 - Velero copies the CNPG and MariaDB data directories file by file while
   the databases run, which is not a consistent copy. The Longhorn backups
   start from a snapshot and are crash-consistent. CNPG can back up to
-  MinIO itself (barman cloud plugin), with point-in-time recovery.
+  Garage itself (barman cloud plugin), with point-in-time recovery.
 - The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
   maintenance job pods) was fixed that afternoon by
   `velero/skip-emptydir-policy.yaml`.
@@ -201,10 +205,31 @@ Findings from 2026-10-25:
 - [ ] Whether the immich library stays in Velero. It stays for now
   (2026-10-25); decide again with the off-site copy (2)
 - [ ] CNPG backups through the barman cloud plugin
-- [ ] Velero's target, MinIO on the NAS, is no longer maintained: no
+- [x] Velero's target, MinIO on the NAS, is no longer maintained: no
   community images since October 2025, maintenance mode since December
-  2025, and the repository is archived. Move Velero to the Garage from 2
+  2025, and the repository is archived. Since 2026-10-26 Velero backs up
-  and switch MinIO off
+  to Garage on the same NAS (`nas/garage`, PR #35). The bucket was copied
+  with rclone (185 GiB in about 6 hours, then a catch-up sync just before
+  the switch), so the existing backups and kopia repositories carried on:
+  after the switch all 55 backups were still listed, and a manual backup
+  of ntfy found the existing repository and used the night's snapshot as
+  its parent. Two things made the NAS slow during the copy: Garage on
+  LMDB with `metadata_fsync`, ~270 disk writes/s while idle (now SQLite,
+  see `nas/garage/garage.toml`), and MinIO's scanner, which kept one disk
+  at 97% busy. The scanner is now at `slowest` (`mc admin config set local
+  scanner speed=slowest` in the MinIO container); it may have slowed
+  Velero's backups to MinIO as well
+- [ ] Switch MinIO off once Velero has run on Garage for a week (from
+  2026-11-02): stop the container, then delete `/volume1/minio` (343 GiB)
+  and the hand-made Secret `velero/cloud-credentials` (MinIO's key). Only
+  MinIO still has the kopia repositories of namespaces that are gone
+  (~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
+  mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
+  whether any of them is worth keeping
+- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
+  2026-10-26 next to a 270 MB database, and going by the folders in
+  `meta/snapshots`, each 6-hourly metadata snapshot takes about 70
+  minutes. Not growing; check again after a week of normal use
 
 ### 6. Monitoring
 
@@ -323,8 +348,10 @@ Counted 2026-10-25 over the 73 running containers:
 
 ### 8. Renovate and CI
 
-10 Renovate PRs are open, the oldest from February, and nothing checks a
+On 2026-10-25 10 Renovate PRs were open, the oldest from February, and
-PR before Argo CD applies it.
+nothing checked a PR before Argo CD applied it. All of them have since been
+merged or closed (below), none are open (checked 2026-10-25), and CI checks
+every PR.
 
 - [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
   every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
@@ -446,7 +473,7 @@ critical / high:
 | tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
 | longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
 | actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
-| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
+| gitea/gitea:1.27.3 | 8 | 43 | now 28.2.0 (PR #26, 2026-10-25); not scanned again |
 | immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
 | towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
 | rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |

freshrss/networkpolicy.yaml

@@ -56,3 +56,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

gatus/networkpolicy.yaml

@@ -83,3 +83,29 @@ spec:
             cidr: 192.168.1.204/32
       ports:
         - port: 16443
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

gitea/networkpolicy.yaml

@@ -55,3 +55,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

immich/networkpolicy.yaml

@@ -73,7 +73,9 @@ spec:
 ---
 # The CNPG operator checks each instance on its status port and connects to
 # PostgreSQL; the instance manager inside each pod watches its Cluster
-# through the API server (the node, port 16443).
+# through the API server (the node, port 16443). The barman cloud sidecar
+# sends the WAL and the base backups to Garage on the NAS
+# (postgresql-backup.yaml).
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -100,3 +102,34 @@ spec:
             cidr: 192.168.1.204/32
       ports:
         - port: 16443
+    - to:
+        - ipBlock:
+            cidr: 192.168.1.99/32
+      ports:
+        - port: 3900
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

immich/postgresql-backup.yaml

@@ -0,0 +1,97 @@
+# Backups of immich's database (albums, faces, the photos' metadata; the
+# photos themselves are files on immich-claim) by CNPG itself, through the
+# barman cloud plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base
+# backup every night, and every WAL segment as soon as it is full or 5
+# minutes old (archive_timeout). Both go to the bucket cnpg-immich in Garage
+# on the NAS (nas/garage), so the database can be restored to any moment of
+# the last 30 days. Velero copies the data directory while PostgreSQL runs,
+# which is not a consistent copy. Only the PostgreSQL 17 cluster; the 14
+# one is about to go.
+# The key is Garage's cnpg-immich, in Vault as kv/cnpg-immich-s3; it can
+# only reach this bucket.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: immich-postgres-17-s3
+  namespace: immich
+spec:
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  refreshInterval: 15m
+  target:
+    name: immich-postgres-17-s3
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        ACCESS_KEY_ID: "{{ .access_key_id }}"
+        ACCESS_SECRET_KEY: "{{ .secret_access_key }}"
+        # Garage rejects requests signed for another region
+        REGION: garage
+  data:
+    - secretKey: access_key_id
+      remoteRef:
+        key: kv/cnpg-immich-s3
+        property: access_key_id
+    - secretKey: secret_access_key
+      remoteRef:
+        key: kv/cnpg-immich-s3
+        property: secret_access_key
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: immich-postgres-17
+  namespace: immich
+  annotations:
+    # The CRD comes with the plugin's own app
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  retentionPolicy: 30d
+  configuration:
+    destinationPath: s3://cnpg-immich/
+    endpointURL: http://192.168.1.99:3900
+    s3Credentials:
+      accessKeyId:
+        name: immich-postgres-17-s3
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: immich-postgres-17-s3
+        key: ACCESS_SECRET_KEY
+      region:
+        name: immich-postgres-17-s3
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  instanceSidecarConfiguration:
+    resources:
+      requests:
+        cpu: 10m
+        memory: 64Mi
+    env:
+      # Newer boto3 adds checksums that S3-compatible stores can reject;
+      # the plugin's docs suggest sending them only when required.
+      - name: AWS_REQUEST_CHECKSUM_CALCULATION
+        value: when_required
+      - name: AWS_RESPONSE_CHECKSUM_VALIDATION
+        value: when_required
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: immich-postgres-17-daily
+  namespace: immich
+spec:
+  cluster:
+    name: immich-postgres-17
+  # 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right
+  # away, so WAL archiving has a base backup to start from.
+  schedule: "0 0 3 * * *"
+  immediate: true
+  backupOwnerReference: self
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

immich/postgresql-cluster-17.yaml

@@ -18,6 +18,13 @@ spec:
     shared_preload_libraries:
       - "vchord.so"
 
+  # WAL archiving and base backups to Garage, see postgresql-backup.yaml
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: immich-postgres-17
+
   # Filled once from the PostgreSQL 14 cluster (immich-postgres) after its
   # embedding columns were turned into real[] and pgvecto.rs was dropped,
   # see docs/upgrade-k8s-1.33.md.

mbgwp/networkpolicy.yaml

@@ -85,3 +85,29 @@ spec:
               kubernetes.io/metadata.name: mariadb-system
       ports:
         - port: 3306
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

nas/garage/README.md

@@ -31,5 +31,12 @@ Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
 `velero/external-secret.yaml`. Without the cluster, `$g key info velero
 --show-secret` on the NAS shows it again.
 
+CNPG's backups (`authentik/postgresql-backup.yaml`,
+`immich/postgresql-backup.yaml`) have a bucket and a key per cluster, so
+neither database can touch the other's backups. Made on 2026-10-26 the
+same way as Velero's, with `cnpg-authentik` and `cnpg-immich` as both
+bucket and key name; the keys are in Vault as `kv/cnpg-authentik-s3` and
+`kv/cnpg-immich-s3`.
+
 Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
 after merging, `docker compose pull && docker compose up -d` on the NAS.

ntfy/networkpolicy.yaml

@@ -65,3 +65,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

renovate.json

@@ -4,6 +4,7 @@
     "config:recommended",
     "customManagers:githubActionsVersions"
   ],
+  "timezone": "Europe/Amsterdam",
   "argocd": {
     "description": "Every app folder; Renovate's regexes are RE2, which has no lookahead, so the other folders are skipped through ignorePaths",
     "managerFilePatterns": [
@@ -88,6 +89,27 @@
         "major"
       ],
       "enabled": false
+    },
+    {
+      "description": "Patch updates of apps that hold little state and are easy to roll back, and of CI's tools, merge themselves once CI passes: three days after the release, in the evening or at the weekend, when the cluster can be reached. Not the databases, operators, Vault, authentik, Longhorn, Velero, immich, Gitea or WordPress. Renovate merges itself; the repo has no branch protection for GitHub's auto-merge to wait on",
+      "matchFileNames": [
+        "ntfy/**",
+        "gatus/**",
+        "freshrss/**",
+        "serpbear/**",
+        "actual-budget/**",
+        ".github/workflows/**"
+      ],
+      "matchUpdateTypes": [
+        "patch"
+      ],
+      "minimumReleaseAge": "3 days",
+      "automerge": true,
+      "platformAutomerge": false,
+      "automergeSchedule": [
+        "* 17-21 * * 1-5",
+        "* 10-21 * * 0,6"
+      ]
     }
   ]
 }

serpbear/networkpolicy.yaml

@@ -54,3 +54,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

vault/networkpolicy.yaml

@@ -66,3 +66,29 @@ spec:
               - 169.254.0.0/16
               - 172.16.0.0/12
               - 192.168.0.0/16
+---
+# cert-manager's HTTP-01 solver, which runs only while a certificate
+# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
+# through Envoy. The policies above let Envoy reach the app's port only,
+# which left the solver unreachable (found 2026-10-26, when ynab's renewal
+# hung).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: acme-http01-solver
+spec:
+  podSelector:
+    matchLabels:
+      acme.cert-manager.io/http01-solver: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy
+      ports:
+        - port: 8089

velero/nas-svc.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: nas-minio-external
-spec:
-  type: ExternalName
-  externalName: 192.168.1.99
-  ports:
-    - port: 9000
-      protocol: TCP

velero/velero-helm.yaml

@@ -27,20 +27,22 @@ spec:
               - mountPath: /target
                 name: plugins
         configuration:
+          # Garage on the NAS (nas/garage). Until 2026-10-26 this was MinIO on
+          # the same NAS; the bucket was copied over with rclone, so the
+          # existing backups and kopia repositories carried on.
           backupStorageLocation:
             - name: default
               bucket: velero-backups
               provider: aws
               config:
-                region: minio
+                region: garage
                 s3ForcePathStyle: true
-                s3Url: http://192.168.1.99:9000
+                s3Url: http://192.168.1.99:3900
-                insecureSkipTLSVerify: "true"
           volumeSnapshotLocation:
             - name: default
               provider: aws
               config:
-                region: minio
+                region: garage
           features: EnableCSI
           # Kopia maintenance ran every hour for each of the 15 repositories,
           # and once a day as a full run of several minutes that writes its
@@ -59,11 +61,8 @@ spec:
                   memoryRequest: 256Mi
                   memoryLimit: 2Gi
         credentials:
-          # Use existing secret created by External Secrets Operator
+          # From Vault, velero/external-secret.yaml
-          # existingSecret: velero-minio-credentials
+          existingSecret: velero-s3-credentials
-          # extraEnvVars:
-          # AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
-          existingSecret: cloud-credentials
         deployNodeAgent: true
         nodeAgent:
           containerSecurityContext: