Compare commits

...
14 Commits
9 changed files with 195 additions and 43 deletions

No files matched your search

+17
View File
@@ -101,6 +101,23 @@ spec:
timeoutSeconds: 5
failureThreshold: 30
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
# crawler asking for 88 pages at once got the container OOMKilled on
# 2026-10-25. With 12 workers a burst waits in the listen backlog
# instead. Listen is the chart's default (containerPort).
apachePortsConfig: |
Listen 8000
<IfModule mpm_prefork_module>
StartServers 4
MinSpareServers 2
MaxSpareServers 6
ServerLimit 12
MaxRequestWorkers 12
MaxConnectionsPerChild 1000
</IfModule>
# Custom PHP configuration
customPhpConfig: |
upload_max_filesize = 64M
+16 -6
View File
@@ -3,7 +3,7 @@
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
# changes:
# - one backup (backups/<name>/) and the kopia repositories of the chosen
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with
# namespaces (kopia/<namespace>/) are copied read-only from the NAS with
# rclone into a Docker volume. A second Velero on the real bucket could
# run kopia maintenance on the same repositories as the cluster's own
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
@@ -19,7 +19,7 @@
# Defaults: the newest Completed daily-backup, and ntfy freshrss
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
#
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
# into a temporary directory, and the copied backup holds every Secret of
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
# cluster (kubeconfig in the printed directory) until you run it with
@@ -33,8 +33,6 @@
set -euo pipefail
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
MINIO=${MINIO:-http://192.168.1.99:9000}
BUCKET=velero-backups
PORT=${PORT:-16443}
K3S=rancher/k3s:v1.34.9-k3s1
RCLONE=rclone/rclone:1.75.1
@@ -73,6 +71,18 @@ wait_for() {
done
}
# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
# request signed for the wrong region (rclone defaults to us-east-1); MinIO
# did so with a bare 400 on HEAD.
if [[ -n ${FROM_VOLUME:-} ]]; then
BUCKET=${BUCKET:-velero-backups}
else
bsl=$(pk -n velero get backupstoragelocations default -o json)
S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
fi
backup=${1:-}
shift || true
namespaces=("$@")
@@ -91,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
else
echo "== copy from the NAS (read-only)"
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env")
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
copy "backups/$backup"
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
+48 -18
View File
@@ -50,8 +50,9 @@ group's ns2: ns1 is stuck on an old serial and still answers
mbg.hensen.io (docs/immich-scouts-online.md).
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
rubenhensen.nl too
- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
`rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
cluster uses hensen.io anymore. hensen.io can expire
### 2. Off-site backups
@@ -99,8 +100,17 @@ friend.
ReadOnly location: data restored byte for byte (same sha256), deployment
available, everything removed afterwards. Two pitfalls found on the way
are in the script's header. Not yet run against the real backups:
- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from
the cluster), then schedule it
- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
volume came back with its backed-up byte count (actual-budget 8.4 MB,
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
MinIO runs region `minio`; the script now reads endpoint, region and
bucket from Velero's location. Copying freshrss's kopia repository took
14 of the ~17 minutes
- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
the cluster itself: the Mac (launchd) or node001, ideally with a
read-only MinIO user
### 3. authentik 2026.2.1 → 2026.8.5
@@ -155,8 +165,8 @@ unseals it.
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
doesn't exist, so the OIDC mount's listing visibility was never set.
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
places outside this cluster; without it the data in Vault is lost
- [x] There is one unseal key (1 share, threshold 1), kept safe outside
this cluster (2026-10-25); without it the data in Vault is lost
- [x] Unsealing is in the SSD install steps
- [ ] After a week without problems: delete the three snapshots
@@ -279,16 +289,29 @@ Counted 2026-10-25 over the 73 running containers:
kube-system
- [ ] Memory limits, once there are daytime numbers: the night ones miss
immich's machine learning at work (it idles at 234Mi)
- [ ] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
2026-10-25, its third restart: a crawler asked for 88 pages and assets
within a few seconds (normally ~6 a minute), Apache started a PHP worker
for each, and the container went past its 1Gi limit (500s first, then
the kill; back within seconds). Unrelated to the NetworkPolicies; memory
sat at ~500Mi before. Fixes, in order: cap Apache's prefork workers
(`MaxRequestWorkers` ~16, so a burst queues instead of filling memory),
a page cache plugin. Rate limiting per visitor in Envoy doesn't work
yet: node001 passes TLS through, so every request arrives from
46.224.26.65; that needs the PROXY protocol from node001's nginx to Envoy
for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
the container went past its 1Gi limit. Unrelated to the
NetworkPolicies. Now capped at 12 workers through the chart's
`apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
with `apache2ctl -t` in the same image first. Replaying the burst (90
page requests, 40 at a time): all 200, at most 13 Apache processes and
572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
peak) instead of a crash
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
cluster between Envoy and WordPress that keeps anonymous GETs for a few
minutes and passes anything with a WordPress cookie through. A page
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
waiting in a burst is queueing; the cap alone already prevents the crash
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
little here, `wordpress-importer` is usually for a one-off import, and
`code-snippets` runs PHP from the dashboard
- [ ] Rate limiting per visitor in Envoy needs the client's address, which
node001 loses: it passes TLS through, so every request arrives from
46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
(ClientTrafficPolicy `enableProxyProtocol`)
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
@@ -350,10 +373,11 @@ PR before Argo CD applies it.
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
its own beforehand, the nightly ones are the way back)
- [ ] Still open: #17 external-secrets 2.13.0 → 2.14.0, no breaking
changes in the notes; every secret goes through it, so merge when
there's time to watch a force-sync. It only changes `targetRevision`, so
it still merges cleanly after #33 added values to the same file
- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
manual rebase (#33 had added values on the next line, which git counts
as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
ExternalSecrets refreshed on a force-sync; the 25th is the generated
`gatus/backup-check-token`, which never refreshes by design
### 9. One node
@@ -369,6 +393,12 @@ step after that, if downtime starts to matter.
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
completed kopia maintenance jobs of repositories that no longer exist
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
volumes over the file paths `data/config.custom.php` and
`data/config-user.custom.php`, so FreshRSS sees two empty directories
there and the volumes hold nothing (0 bytes in every backup). Drop both
mounts and the two PVCs, or make them ConfigMap files if custom config
is wanted
- [ ] `kubectl delete namespace secrets`, empty since its app went
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
+35
View File
@@ -0,0 +1,35 @@
# Garage on the NAS
S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
the cluster. Replaced MinIO, which is no longer maintained. One node, data
in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
its PATH.
## Setup
`garage.toml` here has placeholders for the two secrets; on the NAS they
were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
`openssl rand -base64 32` (`admin_token`), and only exist there.
```
cd /volume1/docker/garage
docker compose up -d
g="docker exec garage /garage"
$g layout assign -z nas -c 800G <node id from $g status>
$g layout apply --version 1
$g bucket create velero-backups
$g key create velero
$g bucket allow --read --write --key velero velero-backups
```
Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
`secret_access_key`) and reaches Velero through
`velero/external-secret.yaml`. Without the cluster, `$g key info velero
--show-secret` on the NAS shows it again.
Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
after merging, `docker compose pull && docker compose up -d` on the NAS.
+12
View File
@@ -0,0 +1,12 @@
services:
garage:
image: dxflrs/garage:v2.5.0
container_name: garage
restart: unless-stopped
ports:
- "3900:3900" # S3 API
- "3903:3903" # admin API
volumes:
- ./garage.toml:/etc/garage.toml:ro
- ./meta:/var/lib/garage/meta
- ./data:/var/lib/garage/data
+29
View File
@@ -0,0 +1,29 @@
# One node on the Synology. Velero's kopia data is encrypted, so
# compression would only cost CPU.
#
# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
# shutdown, and a single node has no other node to repair from. With
# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
# with `garage convert-db`.
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
metadata_fsync = true
data_fsync = true
metadata_auto_snapshot_interval = "6h"
replication_factor = 1
compression_level = "none"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "@RPC_SECRET@"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "@ADMIN_TOKEN@"
+30
View File
@@ -0,0 +1,30 @@
# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
# AWS credentials file format the aws plugin and kopia read.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: velero-s3-credentials
spec:
secretStoreRef:
name: vault-backend
kind: ClusterSecretStore
refreshInterval: 15m
target:
name: velero-s3-credentials
creationPolicy: Owner
template:
engineVersion: v2
data:
cloud: |
[default]
aws_access_key_id={{ .access_key_id }}
aws_secret_access_key={{ .secret_access_key }}
data:
- secretKey: access_key_id
remoteRef:
key: kv/velero-s3
property: access_key_id
- secretKey: secret_access_key
remoteRef:
key: kv/velero-s3
property: secret_access_key
-10
View File
@@ -1,10 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: nas-minio-external
spec:
type: ExternalName
externalName: 192.168.1.99
ports:
- port: 9000
protocol: TCP
+8 -9
View File
@@ -27,20 +27,22 @@ spec:
- mountPath: /target
name: plugins
configuration:
# Garage on the NAS (nas/garage). Until 2026-10-25 this was MinIO on
# the same NAS; the bucket was copied over with rclone, so the
# existing backups and kopia repositories carried on.
backupStorageLocation:
- name: default
bucket: velero-backups
provider: aws
config:
region: minio
region: garage
s3ForcePathStyle: true
s3Url: http://192.168.1.99:9000
insecureSkipTLSVerify: "true"
s3Url: http://192.168.1.99:3900
volumeSnapshotLocation:
- name: default
provider: aws
config:
region: minio
region: garage
features: EnableCSI
# Kopia maintenance ran every hour for each of the 15 repositories,
# and once a day as a full run of several minutes that writes its
@@ -59,11 +61,8 @@ spec:
memoryRequest: 256Mi
memoryLimit: 2Gi
credentials:
# Use existing secret created by External Secrets Operator
# existingSecret: velero-minio-credentials
# extraEnvVars:
# AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
existingSecret: cloud-credentials
# From Vault, velero/external-secret.yaml
existingSecret: velero-s3-credentials
deployNodeAgent: true
nodeAgent:
containerSecurityContext: