Compare commits

..
14 Commits
9 changed files with 195 additions and 43 deletions

No files matched your search

+17
View File
@@ -101,6 +101,23 @@ spec:
timeoutSeconds: 5 timeoutSeconds: 5
failureThreshold: 30 failureThreshold: 30
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
# crawler asking for 88 pages at once got the container OOMKilled on
# 2026-10-25. With 12 workers a burst waits in the listen backlog
# instead. Listen is the chart's default (containerPort).
apachePortsConfig: |
Listen 8000
<IfModule mpm_prefork_module>
StartServers 4
MinSpareServers 2
MaxSpareServers 6
ServerLimit 12
MaxRequestWorkers 12
MaxConnectionsPerChild 1000
</IfModule>
# Custom PHP configuration # Custom PHP configuration
customPhpConfig: | customPhpConfig: |
upload_max_filesize = 64M upload_max_filesize = 64M
+16 -6
View File
@@ -3,7 +3,7 @@
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS # k3s in Docker, to prove the backups can be restored. Nothing on the NAS
# changes: # changes:
# - one backup (backups/<name>/) and the kopia repositories of the chosen # - one backup (backups/<name>/) and the kopia repositories of the chosen
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with # namespaces (kopia/<namespace>/) are copied read-only from the NAS with
# rclone into a Docker volume. A second Velero on the real bucket could # rclone into a Docker volume. A second Velero on the real bucket could
# run kopia maintenance on the same repositories as the cluster's own # run kopia maintenance on the same repositories as the cluster's own
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a # - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
@@ -19,7 +19,7 @@
# Defaults: the newest Completed daily-backup, and ntfy freshrss # Defaults: the newest Completed daily-backup, and ntfy freshrss
# actual-budget. Not gatus: a restored Gatus would start sending alerts. # actual-budget. Not gatus: a restored Gatus would start sending alerts.
# #
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials) # It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
# into a temporary directory, and the copied backup holds every Secret of # into a temporary directory, and the copied backup holds every Secret of
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test # the cluster; all of it is deleted at the end. KEEP=1 keeps the test
# cluster (kubeconfig in the printed directory) until you run it with # cluster (kubeconfig in the printed directory) until you run it with
@@ -33,8 +33,6 @@
set -euo pipefail set -euo pipefail
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config} PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
MINIO=${MINIO:-http://192.168.1.99:9000}
BUCKET=velero-backups
PORT=${PORT:-16443} PORT=${PORT:-16443}
K3S=rancher/k3s:v1.34.9-k3s1 K3S=rancher/k3s:v1.34.9-k3s1
RCLONE=rclone/rclone:1.75.1 RCLONE=rclone/rclone:1.75.1
@@ -73,6 +71,18 @@ wait_for() {
done done
} }
# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
# request signed for the wrong region (rclone defaults to us-east-1); MinIO
# did so with a bare 400 on HEAD.
if [[ -n ${FROM_VOLUME:-} ]]; then
BUCKET=${BUCKET:-velero-backups}
else
bsl=$(pk -n velero get backupstoragelocations default -o json)
S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
fi
backup=${1:-} backup=${1:-}
shift || true shift || true
namespaces=("$@") namespaces=("$@")
@@ -91,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/ docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
else else
echo "== copy from the NAS (read-only)" echo "== copy from the NAS (read-only)"
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d | (umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2} awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env" /aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env") printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; } copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
copy "backups/$backup" copy "backups/$backup"
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
+48 -18
View File
@@ -50,8 +50,9 @@ group's ns2: ns1 is stuck on an old serial and still answers
mbg.hensen.io (docs/immich-scouts-online.md). mbg.hensen.io (docs/immich-scouts-online.md).
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems) - [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers - [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
rubenhensen.nl too `rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
cluster uses hensen.io anymore. hensen.io can expire
### 2. Off-site backups ### 2. Off-site backups
@@ -99,8 +100,17 @@ friend.
ReadOnly location: data restored byte for byte (same sha256), deployment ReadOnly location: data restored byte for byte (same sha256), deployment
available, everything removed afterwards. Two pitfalls found on the way available, everything removed afterwards. Two pitfalls found on the way
are in the script's header. Not yet run against the real backups: are in the script's header. Not yet run against the real backups:
- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from - [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
the cluster), then schedule it daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
volume came back with its backed-up byte count (actual-budget 8.4 MB,
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
MinIO runs region `minio`; the script now reads endpoint, region and
bucket from Velero's location. Copying freshrss's kopia repository took
14 of the ~17 minutes
- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
the cluster itself: the Mac (launchd) or node001, ideally with a
read-only MinIO user
### 3. authentik 2026.2.1 → 2026.8.5 ### 3. authentik 2026.2.1 → 2026.8.5
@@ -155,8 +165,8 @@ unseals it.
- The OIDC job called `vault auth tune -listing_visibility`, a flag that - The OIDC job called `vault auth tune -listing_visibility`, a flag that
doesn't exist, so the OIDC mount's listing visibility was never set. doesn't exist, so the OIDC mount's listing visibility was never set.
Fixed; the job now uses the 2.2.1 CLI instead of 1.15 Fixed; the job now uses the 2.2.1 CLI instead of 1.15
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two - [x] There is one unseal key (1 share, threshold 1), kept safe outside
places outside this cluster; without it the data in Vault is lost this cluster (2026-10-25); without it the data in Vault is lost
- [x] Unsealing is in the SSD install steps - [x] Unsealing is in the SSD install steps
- [ ] After a week without problems: delete the three snapshots - [ ] After a week without problems: delete the three snapshots
@@ -279,16 +289,29 @@ Counted 2026-10-25 over the 73 running containers:
kube-system kube-system
- [ ] Memory limits, once there are daytime numbers: the night ones miss - [ ] Memory limits, once there are daytime numbers: the night ones miss
immich's machine learning at work (it idles at 234Mi) immich's machine learning at work (it idles at 234Mi)
- [ ] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on - [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
2026-10-25, its third restart: a crawler asked for 88 pages and assets 2026-10-25, its third restart: a crawler asked for 88 pages and assets
within a few seconds (normally ~6 a minute), Apache started a PHP worker within a few seconds (normally ~6 a minute), Apache started a PHP worker
for each, and the container went past its 1Gi limit (500s first, then for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
the kill; back within seconds). Unrelated to the NetworkPolicies; memory the container went past its 1Gi limit. Unrelated to the
sat at ~500Mi before. Fixes, in order: cap Apache's prefork workers NetworkPolicies. Now capped at 12 workers through the chart's
(`MaxRequestWorkers` ~16, so a burst queues instead of filling memory), `apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
a page cache plugin. Rate limiting per visitor in Envoy doesn't work with `apache2ctl -t` in the same image first. Replaying the burst (90
yet: node001 passes TLS through, so every request arrives from page requests, 40 at a time): all 200, at most 13 Apache processes and
46.224.26.65; that needs the PROXY protocol from node001's nginx to Envoy 572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
peak) instead of a crash
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
cluster between Envoy and WordPress that keeps anonymous GETs for a few
minutes and passes anything with a WordPress cookie through. A page
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
waiting in a burst is queueing; the cap alone already prevents the crash
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
little here, `wordpress-importer` is usually for a one-off import, and
`code-snippets` runs PHP from the dashboard
- [ ] Rate limiting per visitor in Envoy needs the client's address, which
node001 loses: it passes TLS through, so every request arrives from
46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
(ClientTrafficPolicy `enableProxyProtocol`) (ClientTrafficPolicy `enableProxyProtocol`)
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since - [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`) mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
@@ -350,10 +373,11 @@ PR before Argo CD applies it.
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of 28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
its own beforehand, the nightly ones are the way back) its own beforehand, the nightly ones are the way back)
- [ ] Still open: #17 external-secrets 2.13.0 → 2.14.0, no breaking - [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
changes in the notes; every secret goes through it, so merge when manual rebase (#33 had added values on the next line, which git counts
there's time to watch a force-sync. It only changes `targetRevision`, so as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
it still merges cleanly after #33 added values to the same file ExternalSecrets refreshed on a force-sync; the 25th is the generated
`gatus/backup-check-token`, which never refreshes by design
### 9. One node ### 9. One node
@@ -369,6 +393,12 @@ step after that, if downtime starts to matter.
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15 `backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
completed kopia maintenance jobs of repositories that no longer exist completed kopia maintenance jobs of repositories that no longer exist
(nextcloud, owntracks, sogo, sogo-mail, ttrss) (nextcloud, owntracks, sogo, sogo-mail, ttrss)
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
volumes over the file paths `data/config.custom.php` and
`data/config-user.custom.php`, so FreshRSS sees two empty directories
there and the volumes hold nothing (0 bytes in every backup). Drop both
mounts and the two PVCs, or make them ConfigMap files if custom config
is wanted
- [ ] `kubectl delete namespace secrets`, empty since its app went - [ ] `kubectl delete namespace secrets`, empty since its app went
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and - [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
the weeklies since 08-02. Velero doesn't restore from a Failed backup; the weeklies since 08-02. Velero doesn't restore from a Failed backup;
+35
View File
@@ -0,0 +1,35 @@
# Garage on the NAS
S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
the cluster. Replaced MinIO, which is no longer maintained. One node, data
in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
its PATH.
## Setup
`garage.toml` here has placeholders for the two secrets; on the NAS they
were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
`openssl rand -base64 32` (`admin_token`), and only exist there.
```
cd /volume1/docker/garage
docker compose up -d
g="docker exec garage /garage"
$g layout assign -z nas -c 800G <node id from $g status>
$g layout apply --version 1
$g bucket create velero-backups
$g key create velero
$g bucket allow --read --write --key velero velero-backups
```
Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
`secret_access_key`) and reaches Velero through
`velero/external-secret.yaml`. Without the cluster, `$g key info velero
--show-secret` on the NAS shows it again.
Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
after merging, `docker compose pull && docker compose up -d` on the NAS.
+12
View File
@@ -0,0 +1,12 @@
services:
garage:
image: dxflrs/garage:v2.5.0
container_name: garage
restart: unless-stopped
ports:
- "3900:3900" # S3 API
- "3903:3903" # admin API
volumes:
- ./garage.toml:/etc/garage.toml:ro
- ./meta:/var/lib/garage/meta
- ./data:/var/lib/garage/data
+29
View File
@@ -0,0 +1,29 @@
# One node on the Synology. Velero's kopia data is encrypted, so
# compression would only cost CPU.
#
# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
# shutdown, and a single node has no other node to repair from. With
# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
# with `garage convert-db`.
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
metadata_fsync = true
data_fsync = true
metadata_auto_snapshot_interval = "6h"
replication_factor = 1
compression_level = "none"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "@RPC_SECRET@"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "@ADMIN_TOKEN@"
+30
View File
@@ -0,0 +1,30 @@
# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
# AWS credentials file format the aws plugin and kopia read.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: velero-s3-credentials
spec:
secretStoreRef:
name: vault-backend
kind: ClusterSecretStore
refreshInterval: 15m
target:
name: velero-s3-credentials
creationPolicy: Owner
template:
engineVersion: v2
data:
cloud: |
[default]
aws_access_key_id={{ .access_key_id }}
aws_secret_access_key={{ .secret_access_key }}
data:
- secretKey: access_key_id
remoteRef:
key: kv/velero-s3
property: access_key_id
- secretKey: secret_access_key
remoteRef:
key: kv/velero-s3
property: secret_access_key
-10
View File
@@ -1,10 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: nas-minio-external
spec:
type: ExternalName
externalName: 192.168.1.99
ports:
- port: 9000
protocol: TCP
+8 -9
View File
@@ -27,20 +27,22 @@ spec:
- mountPath: /target - mountPath: /target
name: plugins name: plugins
configuration: configuration:
# Garage on the NAS (nas/garage). Until 2026-10-25 this was MinIO on
# the same NAS; the bucket was copied over with rclone, so the
# existing backups and kopia repositories carried on.
backupStorageLocation: backupStorageLocation:
- name: default - name: default
bucket: velero-backups bucket: velero-backups
provider: aws provider: aws
config: config:
region: minio region: garage
s3ForcePathStyle: true s3ForcePathStyle: true
s3Url: http://192.168.1.99:9000 s3Url: http://192.168.1.99:3900
insecureSkipTLSVerify: "true"
volumeSnapshotLocation: volumeSnapshotLocation:
- name: default - name: default
provider: aws provider: aws
config: config:
region: minio region: garage
features: EnableCSI features: EnableCSI
# Kopia maintenance ran every hour for each of the 15 repositories, # Kopia maintenance ran every hour for each of the 15 repositories,
# and once a day as a full run of several minutes that writes its # and once a day as a full run of several minutes that writes its
@@ -59,11 +61,8 @@ spec:
memoryRequest: 256Mi memoryRequest: 256Mi
memoryLimit: 2Gi memoryLimit: 2Gi
credentials: credentials:
# Use existing secret created by External Secrets Operator # From Vault, velero/external-secret.yaml
# existingSecret: velero-minio-credentials existingSecret: velero-s3-credentials
# extraEnvVars:
# AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
existingSecret: cloud-credentials
deployNodeAgent: true deployNodeAgent: true
nodeAgent: nodeAgent:
containerSecurityContext: containerSecurityContext: