Compare commits

..
Author SHA1 Message Date
renovate[bot] d7daae80fd chore(deps): update freshrss/freshrss docker tag to v1.30.4 2026-10-25 16:42:09 +00:00
7 changed files with 62 additions and 97 deletions

No files matched your search

+8 -8
View File
@@ -3,7 +3,7 @@
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS # k3s in Docker, to prove the backups can be restored. Nothing on the NAS
# changes: # changes:
# - one backup (backups/<name>/) and the kopia repositories of the chosen # - one backup (backups/<name>/) and the kopia repositories of the chosen
# namespaces (kopia/<namespace>/) are copied read-only from the NAS with # namespaces (kopia/<namespace>/) are copied read-only from MinIO with
# rclone into a Docker volume. A second Velero on the real bucket could # rclone into a Docker volume. A second Velero on the real bucket could
# run kopia maintenance on the same repositories as the cluster's own # run kopia maintenance on the same repositories as the cluster's own
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a # - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
@@ -19,7 +19,7 @@
# Defaults: the newest Completed daily-backup, and ntfy freshrss # Defaults: the newest Completed daily-backup, and ntfy freshrss
# actual-budget. Not gatus: a restored Gatus would start sending alerts. # actual-budget. Not gatus: a restored Gatus would start sending alerts.
# #
# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials) # It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
# into a temporary directory, and the copied backup holds every Secret of # into a temporary directory, and the copied backup holds every Secret of
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test # the cluster; all of it is deleted at the end. KEEP=1 keeps the test
# cluster (kubeconfig in the printed directory) until you run it with # cluster (kubeconfig in the printed directory) until you run it with
@@ -71,14 +71,14 @@ wait_for() {
done done
} }
# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a # Endpoint, region and bucket as Velero itself uses them. MinIO answers a
# request signed for the wrong region (rclone defaults to us-east-1); MinIO # request signed for the wrong region (rclone defaults to us-east-1) with a
# did so with a bare 400 on HEAD. # bare 400 on HEAD.
if [[ -n ${FROM_VOLUME:-} ]]; then if [[ -n ${FROM_VOLUME:-} ]]; then
BUCKET=${BUCKET:-velero-backups} BUCKET=${BUCKET:-velero-backups}
else else
bsl=$(pk -n velero get backupstoragelocations default -o json) bsl=$(pk -n velero get backupstoragelocations default -o json)
S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")} MINIO=${MINIO:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")} REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")} BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
fi fi
@@ -101,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/ docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
else else
echo "== copy from the NAS (read-only)" echo "== copy from the NAS (read-only)"
(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d | (umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2} awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env" /aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env") printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$MINIO" "$REGION" >> "$WORK/nas.env")
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; } copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
copy "backups/$backup" copy "backups/$backup"
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
+31 -59
View File
@@ -10,11 +10,11 @@ One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held. SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn 35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to Garage NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
(MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90 days). 01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert and s2 (home) watch each other and alert through ntfy; s2 also gets an
through ntfy; s2 also gets an hourly report from inside the cluster on the hourly report from inside the cluster on the backups, Vault's seal, the
backups, Vault's seal, the Argo CD apps and the CNPG clusters. Argo CD apps and the CNPG clusters.
Every app namespace has NetworkPolicies, and CI renders and validates all Every app namespace has NetworkPolicies, and CI renders and validates all
apps on every PR. node001 runs mail and tunnels the public hosts home. apps on every PR. node001 runs mail and tunnels the public hosts home.
@@ -38,8 +38,7 @@ docs/longhorn-second-ssd.md:
2026-10-25). The snap is held, so microk8s gets no updates at all until 2026-10-25). The snap is held, so microk8s gets no updates at all until
someone refreshes it someone refreshes it
- [ ] Second SSD, docs/longhorn-second-ssd.md - [ ] Second SSD, docs/longhorn-second-ssd.md
- [x] `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML (PR #39, 2026-10-25; - [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
the models are cached on a volume)
## Next, in order ## Next, in order
@@ -64,25 +63,24 @@ belong to the group.
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
friend. friend.
- [ ] Ours to his bucket: `rclone sync` from Garage's `velero-backups` - [ ] Ours to his bucket: Hyper Backup on the Synology, destination "S3
(~190 GB) to his bucket through an rclone `crypt` remote, in Docker on Storage" with his endpoint as a custom server. Client-side encryption
the NAS like the copy from MinIO (see 5). Encrypted on our side: the on: the kopia repositories use Velero's default password
kopia repositories use Velero's default password (`static-passw0rd`), (`static-passw0rd`), so without it he could read everything, the
so without it he could read everything, the group's photos included. group's photos included. Keep the encryption key next to the Vault
Keep the crypt passwords next to the Vault unseal key. Daily after the unseal key. Source: the shared folder with MinIO's `velero-backups`
Velero backup, ntfy on failure. Hyper Backup, the earlier plan, backs up bucket (~150-200 GB). Daily after the Velero backup, Smart Recycle,
shared folders; Garage keeps its objects as blocks plus a metadata mail on failure
database, and a copy of that folder taken while Garage runs is not
consistent
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB - [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
now, 500 GB to grow), and versioning or object lock on the bucket, so now, 500 GB to grow), and versioning or object lock on the bucket, so
whoever takes over the NAS can't delete the off-site copy as well whoever takes over the NAS can't delete the off-site copy as well
- [ ] His backups with us: a bucket and key for him, with a quota, in the - [ ] His backups with us: Garage (not MinIO, see 5) in Container Manager
Garage on the NAS (`nas/garage`, see 5). Only the S3 API public, as on the Synology, in its own shared folder, with a bucket and key for him
`s3.rubenhensen.nl` through node001 and Envoy with a cert-manager and a quota. Only the S3 API public, as `s3.rubenhensen.nl` through
certificate; the admin API stays on the LAN. The cluster side (DNS, node001 and Envoy with a cert-manager certificate; the admin API stays
node001 tunnel, Certificate, HTTPRoute) goes in git. Needed: how much on the LAN. The cluster side (DNS, node001 tunnel, Certificate,
space he wants, free space on the NAS HTTPRoute) goes in git. Needed: how much space he wants, free space on
the NAS
- [ ] Decide which copy of the immich library goes off-site (see 5) - [ ] Decide which copy of the immich library goes off-site (see 5)
- [ ] A restore test that runs on a schedule. `backup-testing/` predates - [ ] A restore test that runs on a schedule. `backup-testing/` predates
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth. Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
@@ -110,12 +108,9 @@ friend.
MinIO runs region `minio`; the script now reads endpoint, region and MinIO runs region `minio`; the script now reads endpoint, region and
bucket from Velero's location. Copying freshrss's kopia repository took bucket from Velero's location. Copying freshrss's kopia repository took
14 of the ~17 minutes 14 of the ~17 minutes
- [x] Ran it again on 2026-10-26 against the copy in Garage - [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
(daily-backup-20261026010028, same namespaces), before Velero switched the cluster itself: the Mac (launchd) or node001, ideally with a
over (see 5): passed, in 5 minutes instead of 17 read-only MinIO user
- [ ] Run it on a schedule. It needs Docker and an S3 key, so not in the
cluster itself: the Mac (launchd) or node001, ideally with a read-only
Garage key (`garage key create`, `garage bucket allow --read`)
### 3. authentik 2026.2.1 → 2026.8.5 ### 3. authentik 2026.2.1 → 2026.8.5
@@ -198,7 +193,7 @@ Findings from 2026-10-25:
- Velero copies the CNPG and MariaDB data directories file by file while - Velero copies the CNPG and MariaDB data directories file by file while
the databases run, which is not a consistent copy. The Longhorn backups the databases run, which is not a consistent copy. The Longhorn backups
start from a snapshot and are crash-consistent. CNPG can back up to start from a snapshot and are crash-consistent. CNPG can back up to
Garage itself (barman cloud plugin), with point-in-time recovery. MinIO itself (barman cloud plugin), with point-in-time recovery.
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own - The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
maintenance job pods) was fixed that afternoon by maintenance job pods) was fixed that afternoon by
`velero/skip-emptydir-policy.yaml`. `velero/skip-emptydir-policy.yaml`.
@@ -206,31 +201,10 @@ Findings from 2026-10-25:
- [ ] Whether the immich library stays in Velero. It stays for now - [ ] Whether the immich library stays in Velero. It stays for now
(2026-10-25); decide again with the off-site copy (2) (2026-10-25); decide again with the off-site copy (2)
- [ ] CNPG backups through the barman cloud plugin - [ ] CNPG backups through the barman cloud plugin
- [x] Velero's target, MinIO on the NAS, is no longer maintained: no - [ ] Velero's target, MinIO on the NAS, is no longer maintained: no
community images since October 2025, maintenance mode since December community images since October 2025, maintenance mode since December
2025, and the repository is archived. Since 2026-10-26 Velero backs up 2025, and the repository is archived. Move Velero to the Garage from 2
to Garage on the same NAS (`nas/garage`, PR #35). The bucket was copied and switch MinIO off
with rclone (185 GiB in about 6 hours, then a catch-up sync just before
the switch), so the existing backups and kopia repositories carried on:
after the switch all 55 backups were still listed, and a manual backup
of ntfy found the existing repository and used the night's snapshot as
its parent. Two things made the NAS slow during the copy: Garage on
LMDB with `metadata_fsync`, ~270 disk writes/s while idle (now SQLite,
see `nas/garage/garage.toml`), and MinIO's scanner, which kept one disk
at 97% busy. The scanner is now at `slowest` (`mc admin config set local
scanner speed=slowest` in the MinIO container); it may have slowed
Velero's backups to MinIO as well
- [ ] Switch MinIO off once Velero has run on Garage for a week (from
2026-11-02): stop the container, then delete `/volume1/minio` (343 GiB)
and the hand-made Secret `velero/cloud-credentials` (MinIO's key). Only
MinIO still has the kopia repositories of namespaces that are gone
(~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
whether any of them is worth keeping
- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
2026-10-26 next to a 270 MB database, and going by the folders in
`meta/snapshots`, each 6-hourly metadata snapshot takes about 70
minutes. Not growing; check again after a week of normal use
### 6. Monitoring ### 6. Monitoring
@@ -349,10 +323,8 @@ Counted 2026-10-25 over the 73 running containers:
### 8. Renovate and CI ### 8. Renovate and CI
On 2026-10-25 10 Renovate PRs were open, the oldest from February, and 10 Renovate PRs are open, the oldest from February, and nothing checks a
nothing checked a PR before Argo CD applied it. All of them have since been PR before Argo CD applies it.
merged or closed (below), none are open (checked 2026-10-25), and CI checks
every PR.
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on - [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
@@ -474,7 +446,7 @@ critical / high:
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag | | tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below | | longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it | | actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
| gitea/gitea:1.27.3 | 8 | 43 | now 28.2.0 (PR #26, 2026-10-25); not scanned again | | gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release | | immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release | | towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image | | rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
+1 -1
View File
@@ -74,7 +74,7 @@ spec:
value: Europe/Amsterdam value: Europe/Amsterdam
# 1.30.3 ships PHP 8.4.27, which fixes CVE-2026-109618 and # 1.30.3 ships PHP 8.4.27, which fixes CVE-2026-109618 and
# CVE-2026-109619. # CVE-2026-109619.
image: freshrss/freshrss:1.30.3 image: freshrss/freshrss:1.30.4
name: freshrss name: freshrss
ports: ports:
- containerPort: 80 - containerPort: 80
+3 -14
View File
@@ -127,9 +127,6 @@ spec:
TRANSFORMERS_CACHE: /cache TRANSFORMERS_CACHE: /cache
HF_XET_CACHE: /cache/huggingface-xet HF_XET_CACHE: /cache/huggingface-xet
MPLCONFIGDIR: /cache/matplotlib-config MPLCONFIGDIR: /cache/matplotlib-config
# The optimized models of v3.3.0: faster, less memory,
# and the same results, so no jobs to run again.
MACHINE_LEARNING_MODEL_REVISION: v2
# Trust store without the compromised eMudhra # Trust store without the compromised eMudhra
# intermediates; requests would otherwise use certifi. # intermediates; requests would otherwise use certifi.
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
@@ -142,19 +139,11 @@ spec:
globalMounts: globalMounts:
- path: /etc/ssl/certs - path: /etc/ssl/certs
readOnly: true readOnly: true
# The models and their one-off preparation, kept across restarts.
# Downloaded again if lost, so no backups: the no-backup group
# keeps it out of Longhorn's default one (backup-daily), and
# Velero skips that label too (velero/skip-emptydir-policy.yaml).
cache: cache:
enabled: true enabled: true
type: persistentVolumeClaim # Optional: to avoid downloading the ML models every start, use
storageClass: longhorn # a persistentVolumeClaim with size, accessMode and storageClass
accessMode: ReadWriteOnce type: emptyDir
size: 10Gi
labels:
recurring-job.longhorn.io/source: enabled
recurring-job-group.longhorn.io/no-backup: enabled
destination: destination:
server: https://kubernetes.default.svc server: https://kubernetes.default.svc
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v1
kind: Service
metadata:
name: nas-minio-external
spec:
type: ExternalName
externalName: 192.168.1.99
ports:
- port: 9000
protocol: TCP
-7
View File
@@ -1,8 +1,6 @@
# Volume policy for the backup schedules. emptyDir volumes hold nothing # Volume policy for the backup schedules. emptyDir volumes hold nothing
# worth keeping, and backing them up fails when the pod is gone before its # worth keeping, and backing them up fails when the pod is gone before its
# volume is read, as happens with Velero's own repository maintenance jobs. # volume is read, as happens with Velero's own repository maintenance jobs.
# PVCs in Longhorn's no-backup group (caches, such as immich's ML models)
# are skipped too.
apiVersion: v1 apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:
@@ -16,8 +14,3 @@ data:
- emptyDir - emptyDir
action: action:
type: skip type: skip
- conditions:
pvcLabels:
recurring-job-group.longhorn.io/no-backup: enabled
action:
type: skip
+9 -8
View File
@@ -27,22 +27,20 @@ spec:
- mountPath: /target - mountPath: /target
name: plugins name: plugins
configuration: configuration:
# Garage on the NAS (nas/garage). Until 2026-10-26 this was MinIO on
# the same NAS; the bucket was copied over with rclone, so the
# existing backups and kopia repositories carried on.
backupStorageLocation: backupStorageLocation:
- name: default - name: default
bucket: velero-backups bucket: velero-backups
provider: aws provider: aws
config: config:
region: garage region: minio
s3ForcePathStyle: true s3ForcePathStyle: true
s3Url: http://192.168.1.99:3900 s3Url: http://192.168.1.99:9000
insecureSkipTLSVerify: "true"
volumeSnapshotLocation: volumeSnapshotLocation:
- name: default - name: default
provider: aws provider: aws
config: config:
region: garage region: minio
features: EnableCSI features: EnableCSI
# Kopia maintenance ran every hour for each of the 15 repositories, # Kopia maintenance ran every hour for each of the 15 repositories,
# and once a day as a full run of several minutes that writes its # and once a day as a full run of several minutes that writes its
@@ -61,8 +59,11 @@ spec:
memoryRequest: 256Mi memoryRequest: 256Mi
memoryLimit: 2Gi memoryLimit: 2Gi
credentials: credentials:
# From Vault, velero/external-secret.yaml # Use existing secret created by External Secrets Operator
existingSecret: velero-s3-credentials # existingSecret: velero-minio-credentials
# extraEnvVars:
# AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
existingSecret: cloud-credentials
deployNodeAgent: true deployNodeAgent: true
nodeAgent: nodeAgent:
containerSecurityContext: containerSecurityContext: