diff --git a/apps/templates/plugin-barman-cloud-helm.yaml b/apps/templates/plugin-barman-cloud-helm.yaml new file mode 100644 index 0000000..1d94bbe --- /dev/null +++ b/apps/templates/plugin-barman-cloud-helm.yaml @@ -0,0 +1,32 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: plugin-barman-cloud + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + chart: plugin-barman-cloud + repoURL: https://cloudnative-pg.github.io/charts + targetRevision: 0.8.1 + helm: + values: | + # CNPG's barman cloud plugin, which backs the PostgreSQL clusters + # up to Garage on the NAS (authentik/postgresql-backup.yaml, + # immich/postgresql-backup.yaml). It has to run in the operator's + # namespace; the chart gets its TLS certificates from cert-manager. + resources: + requests: + cpu: 10m + memory: 64Mi + destination: + server: https://kubernetes.default.svc + namespace: cnpg-system + syncPolicy: + syncOptions: + - ServerSideApply=true + automated: + prune: true + selfHeal: true diff --git a/authentik/networkpolicy.yaml b/authentik/networkpolicy.yaml index f9f75b4..c78bae2 100644 --- a/authentik/networkpolicy.yaml +++ b/authentik/networkpolicy.yaml @@ -108,7 +108,8 @@ spec: --- # The CNPG operator checks each instance on its status port and connects to # PostgreSQL; the instance manager inside each pod watches its Cluster -# through the API server. +# through the API server. The barman cloud sidecar sends the WAL and the +# base backups to Garage on the NAS (postgresql-backup.yaml). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -135,6 +136,11 @@ spec: cidr: 192.168.1.204/32 ports: - port: 16443 + - to: + - ipBlock: + cidr: 192.168.1.99/32 + ports: + - port: 3900 --- # cert-manager's HTTP-01 solver, which runs only while a certificate # renews: Let's Encrypt and cert-manager's own self check reach it on 8089 diff --git a/authentik/postgresql-backup.yaml b/authentik/postgresql-backup.yaml new file mode 100644 index 0000000..f47b1c4 --- /dev/null +++ b/authentik/postgresql-backup.yaml @@ -0,0 +1,92 @@ +# Backups of authentik's database by CNPG itself, through the barman cloud +# plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base backup every +# night, and every WAL segment as soon as it is full or 5 minutes old +# (archive_timeout). Both go to the bucket cnpg-authentik in Garage on the +# NAS (nas/garage), so the database can be restored to any moment of the +# last 30 days. Velero copies the data directory while PostgreSQL runs, +# which is not a consistent copy. +# The key is Garage's cnpg-authentik, in Vault as kv/cnpg-authentik-s3; it +# can only reach this bucket. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: authentik-postgres-s3 +spec: + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + refreshInterval: 15m + target: + name: authentik-postgres-s3 + creationPolicy: Owner + template: + engineVersion: v2 + data: + ACCESS_KEY_ID: "{{ .access_key_id }}" + ACCESS_SECRET_KEY: "{{ .secret_access_key }}" + # Garage rejects requests signed for another region + REGION: garage + data: + - secretKey: access_key_id + remoteRef: + key: kv/cnpg-authentik-s3 + property: access_key_id + - secretKey: secret_access_key + remoteRef: + key: kv/cnpg-authentik-s3 + property: secret_access_key +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: authentik-postgres + annotations: + # The CRD comes with the plugin's own app + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + retentionPolicy: 30d + configuration: + destinationPath: s3://cnpg-authentik/ + endpointURL: http://192.168.1.99:3900 + s3Credentials: + accessKeyId: + name: authentik-postgres-s3 + key: ACCESS_KEY_ID + secretAccessKey: + name: authentik-postgres-s3 + key: ACCESS_SECRET_KEY + region: + name: authentik-postgres-s3 + key: REGION + wal: + compression: gzip + data: + compression: gzip + instanceSidecarConfiguration: + resources: + requests: + cpu: 10m + memory: 64Mi + env: + # Newer boto3 adds checksums that S3-compatible stores can reject; + # the plugin's docs suggest sending them only when required. + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + - name: AWS_RESPONSE_CHECKSUM_VALIDATION + value: when_required +--- +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: authentik-postgres-daily +spec: + cluster: + name: authentik-postgres + # 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right + # away, so WAL archiving has a base backup to start from. + schedule: "0 0 3 * * *" + immediate: true + backupOwnerReference: self + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io diff --git a/authentik/postgresql-cluster.yaml b/authentik/postgresql-cluster.yaml index 3526b91..ee87f56 100644 --- a/authentik/postgresql-cluster.yaml +++ b/authentik/postgresql-cluster.yaml @@ -8,6 +8,12 @@ spec: # start an in-place major upgrade. CNPG filled this in at creation. imageName: ghcr.io/cloudnative-pg/postgresql:17.4 primaryUpdateStrategy: unsupervised + # WAL archiving and base backups to Garage, see postgresql-backup.yaml + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: authentik-postgres resources: requests: cpu: 50m diff --git a/immich/networkpolicy.yaml b/immich/networkpolicy.yaml index c292815..5683588 100644 --- a/immich/networkpolicy.yaml +++ b/immich/networkpolicy.yaml @@ -73,7 +73,9 @@ spec: --- # The CNPG operator checks each instance on its status port and connects to # PostgreSQL; the instance manager inside each pod watches its Cluster -# through the API server (the node, port 16443). +# through the API server (the node, port 16443). The barman cloud sidecar +# sends the WAL and the base backups to Garage on the NAS +# (postgresql-backup.yaml). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -100,6 +102,11 @@ spec: cidr: 192.168.1.204/32 ports: - port: 16443 + - to: + - ipBlock: + cidr: 192.168.1.99/32 + ports: + - port: 3900 --- # cert-manager's HTTP-01 solver, which runs only while a certificate # renews: Let's Encrypt and cert-manager's own self check reach it on 8089 diff --git a/immich/postgresql-backup.yaml b/immich/postgresql-backup.yaml new file mode 100644 index 0000000..cbd42eb --- /dev/null +++ b/immich/postgresql-backup.yaml @@ -0,0 +1,97 @@ +# Backups of immich's database (albums, faces, the photos' metadata; the +# photos themselves are files on immich-claim) by CNPG itself, through the +# barman cloud plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base +# backup every night, and every WAL segment as soon as it is full or 5 +# minutes old (archive_timeout). Both go to the bucket cnpg-immich in Garage +# on the NAS (nas/garage), so the database can be restored to any moment of +# the last 30 days. Velero copies the data directory while PostgreSQL runs, +# which is not a consistent copy. Only the PostgreSQL 17 cluster; the 14 +# one is about to go. +# The key is Garage's cnpg-immich, in Vault as kv/cnpg-immich-s3; it can +# only reach this bucket. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: immich-postgres-17-s3 + namespace: immich +spec: + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + refreshInterval: 15m + target: + name: immich-postgres-17-s3 + creationPolicy: Owner + template: + engineVersion: v2 + data: + ACCESS_KEY_ID: "{{ .access_key_id }}" + ACCESS_SECRET_KEY: "{{ .secret_access_key }}" + # Garage rejects requests signed for another region + REGION: garage + data: + - secretKey: access_key_id + remoteRef: + key: kv/cnpg-immich-s3 + property: access_key_id + - secretKey: secret_access_key + remoteRef: + key: kv/cnpg-immich-s3 + property: secret_access_key +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: immich-postgres-17 + namespace: immich + annotations: + # The CRD comes with the plugin's own app + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + retentionPolicy: 30d + configuration: + destinationPath: s3://cnpg-immich/ + endpointURL: http://192.168.1.99:3900 + s3Credentials: + accessKeyId: + name: immich-postgres-17-s3 + key: ACCESS_KEY_ID + secretAccessKey: + name: immich-postgres-17-s3 + key: ACCESS_SECRET_KEY + region: + name: immich-postgres-17-s3 + key: REGION + wal: + compression: gzip + data: + compression: gzip + instanceSidecarConfiguration: + resources: + requests: + cpu: 10m + memory: 64Mi + env: + # Newer boto3 adds checksums that S3-compatible stores can reject; + # the plugin's docs suggest sending them only when required. + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + - name: AWS_RESPONSE_CHECKSUM_VALIDATION + value: when_required +--- +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: immich-postgres-17-daily + namespace: immich +spec: + cluster: + name: immich-postgres-17 + # 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right + # away, so WAL archiving has a base backup to start from. + schedule: "0 0 3 * * *" + immediate: true + backupOwnerReference: self + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io diff --git a/immich/postgresql-cluster-17.yaml b/immich/postgresql-cluster-17.yaml index 72e5199..eb29418 100644 --- a/immich/postgresql-cluster-17.yaml +++ b/immich/postgresql-cluster-17.yaml @@ -18,6 +18,13 @@ spec: shared_preload_libraries: - "vchord.so" + # WAL archiving and base backups to Garage, see postgresql-backup.yaml + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: immich-postgres-17 + # Filled once from the PostgreSQL 14 cluster (immich-postgres) after its # embedding columns were turned into real[] and pgvecto.rs was dropped, # see docs/upgrade-k8s-1.33.md. diff --git a/nas/garage/README.md b/nas/garage/README.md index e2346d8..130cd5b 100644 --- a/nas/garage/README.md +++ b/nas/garage/README.md @@ -31,5 +31,12 @@ Velero's key is in Vault as `kv/velero-s3` (`access_key_id`, `velero/external-secret.yaml`. Without the cluster, `$g key info velero --show-secret` on the NAS shows it again. +CNPG's backups (`authentik/postgresql-backup.yaml`, +`immich/postgresql-backup.yaml`) have a bucket and a key per cluster, so +neither database can touch the other's backups. Made on 2026-10-26 the +same way as Velero's, with `cnpg-authentik` and `cnpg-immich` as both +bucket and key name; the keys are in Vault as `kv/cnpg-authentik-s3` and +`kv/cnpg-immich-s3`. + Renovate updates the image tag in `compose.yaml`, but nothing deploys it: after merging, `docker compose pull && docker compose up -d` on the NAS.