Compare commits

..
Author SHA1 Message Date
renovate[bot] b6077ae917 chore(deps): update helm chart external-secrets to v2.14.0 2026-10-25 03:06:31 +00:00
36 changed files with 71 additions and 1052 deletions

No files matched your search

-8
View File
@@ -99,10 +99,6 @@ spec:
- name: REQUESTS_CA_BUNDLE
value: /etc/ssl/certs/ca-certificates.crt
server:
resources:
requests:
cpu: 70m
memory: 560Mi
volumes:
- name: patched-ca-bundle
configMap:
@@ -188,10 +184,6 @@ spec:
mountPath: /etc/ssl/certs
readOnly: true
worker:
resources:
requests:
cpu: 100m
memory: 336Mi
volumes:
- name: patched-ca-bundle
configMap:
+1 -15
View File
@@ -12,7 +12,7 @@ spec:
project: default
source:
repoURL: https://charts.jetstack.io
targetRevision: v1.21.3
targetRevision: v1.21.2
chart: cert-manager
helm:
parameters:
@@ -34,20 +34,6 @@ spec:
- name: patched-ca-bundle
mountPath: /etc/ssl/certs
readOnly: true
resources:
requests:
cpu: 10m
memory: 48Mi
cainjector:
resources:
requests:
cpu: 10m
memory: 80Mi
webhook:
resources:
requests:
cpu: 10m
memory: 32Mi
syncPolicy:
syncOptions:
- CreateNamespace=true
-4
View File
@@ -24,10 +24,6 @@ spec:
additionalArgs:
- --leader-lease-duration=60
- --leader-renew-deadline=40
resources:
requests:
cpu: 10m
memory: 48Mi
destination:
server: https://kubernetes.default.svc
namespace: cnpg-system
-16
View File
@@ -18,22 +18,6 @@ spec:
- chart: external-secrets
repoURL: https://charts.external-secrets.io
targetRevision: 2.14.0
helm:
values: |
resources:
requests:
cpu: 20m
memory: 64Mi
certController:
resources:
requests:
cpu: 10m
memory: 64Mi
webhook:
resources:
requests:
cpu: 10m
memory: 32Mi
destination:
server: https://kubernetes.default.svc
namespace: external-secrets
+1 -18
View File
@@ -12,7 +12,7 @@ spec:
project: default
source:
repoURL: https://groundhog2k.github.io/helm-charts/
targetRevision: 0.16.7
targetRevision: 0.15.0
chart: wordpress
helm:
values: |
@@ -101,23 +101,6 @@ spec:
timeoutSeconds: 5
failureThreshold: 30
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
# crawler asking for 88 pages at once got the container OOMKilled on
# 2026-10-25. With 12 workers a burst waits in the listen backlog
# instead. Listen is the chart's default (containerPort).
apachePortsConfig: |
Listen 8000
<IfModule mpm_prefork_module>
StartServers 4
MinSpareServers 2
MaxSpareServers 6
ServerLimit 12
MaxRequestWorkers 12
MaxConnectionsPerChild 1000
</IfModule>
# Custom PHP configuration
customPhpConfig: |
upload_max_filesize = 64M
-34
View File
@@ -1,34 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: metrics-server
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
chart: metrics-server
repoURL: https://kubernetes-sigs.github.io/metrics-server/
targetRevision: 3.14.0
helm:
values: |
# Resource usage for kubectl top and for sizing requests and limits.
# It keeps everything in memory, so it adds no writes to the SSD.
args:
# microk8s's kubelet serves a self-signed certificate; its own
# metrics-server addon passes the same flag.
- --kubelet-insecure-tls
resources:
requests:
cpu: 20m
memory: 64Mi
limits:
memory: 256Mi
destination:
server: https://kubernetes.default.svc
namespace: kube-system
syncPolicy:
automated:
prune: true
selfHeal: true
+1 -1
View File
@@ -7,7 +7,7 @@ spec:
project: default
source:
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
targetRevision: v8.7.0
targetRevision: v8.2.1
path: deploy/kubernetes/snapshot-controller
kustomize:
patches:
+1 -2
View File
@@ -21,5 +21,4 @@ patches:
- path: overlays/argocd-cm.patch.yaml
- path: overlays/argocd-rbac-cm.patch.yaml
- path: overlays/argocd-cmd-params-cm.patch.yaml
- path: overlays/patched-ca-bundle.patch.yaml
- path: overlays/resources.patch.yaml
- path: overlays/patched-ca-bundle.patch.yaml
-100
View File
@@ -1,100 +0,0 @@
# Requests from what the components used on 2026-10-25 (kubectl top every
# 5 minutes; highest value plus 10%). No limits until there are daytime
# numbers, see docs/roadmap.md.
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: argocd-application-controller
spec:
template:
spec:
containers:
- name: argocd-application-controller
resources:
requests:
cpu: 150m
memory: 704Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-repo-server
spec:
template:
spec:
containers:
- name: argocd-repo-server
resources:
requests:
cpu: 100m
memory: 128Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-server
spec:
template:
spec:
containers:
- name: argocd-server
resources:
requests:
cpu: 10m
memory: 64Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-redis
spec:
template:
spec:
containers:
- name: redis
resources:
requests:
cpu: 20m
memory: 96Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-applicationset-controller
spec:
template:
spec:
containers:
- name: argocd-applicationset-controller
resources:
requests:
cpu: 10m
memory: 48Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-notifications-controller
spec:
template:
spec:
containers:
- name: argocd-notifications-controller
resources:
requests:
cpu: 10m
memory: 48Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-dex-server
spec:
template:
spec:
containers:
- name: dex
resources:
requests:
cpu: 10m
memory: 32Mi
+3 -26
View File
@@ -14,11 +14,7 @@ Authentik is deployed as the central identity provider, providing OIDC and LDAP
- `blueprint-argocd-oidc.yaml` — OIDC provider for ArgoCD
- `blueprint-freshrss-oidc.yaml` — OIDC provider for FreshRSS
- `blueprint-actualbudget-proxy.yaml` — Proxy provider for Actual Budget
- `blueprint-longhorn-oidc.yaml` — OIDC provider for Longhorn (Envoy Gateway SecurityPolicy)
- `blueprint-scouting.yaml` — immich's login with Scouts Online, see docs/immich-scouts-online.md
- `blueprint-session-duration.yaml` — Session lifetime of the default authentication flow
- **NetworkPolicies** (`networkpolicy.yaml`) — the pods reach each other, DNS and the internet,
not the LAN or other namespaces; the worker and the database also reach the API server
## Session lifetime
@@ -34,23 +30,9 @@ Apache mod_auth_openidc refreshes on every request.
## LDAP Outpost
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by
**Stalwart** (NixOS on node001), which connects via `ldap.rubenhensen.nl:389`.
The outpost itself talks to the server through `https://authentik.rubenhensen.nl/`.
Authentik's worker deploys the outpost and keeps its version equal to the
server's. After the upgrades to 2026.5 and 2026.8 it didn't follow on its
own; this rolls it over:
```bash
kubectl -n authentik exec deploy/authentik-worker -- ak shell -c '
from authentik.outposts.models import Outpost
from authentik.outposts.tasks import outpost_controller
outpost_controller.send(Outpost.objects.get(name="LDAP outpost").pk)'
```
Upgrades go one release at a time (2026.2 → 2026.5 → 2026.8); since 2026.8
authentik refuses to skip one.
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by:
- **SOGo-mail** (in-cluster) — connects via `ak-outpost-ldap-outpost.authentik.svc.cluster.local:3389`
- **Stalwart** (NixOS, external) — connects via `ldap.rubenhensen.nl:389`
### NodePort Service
@@ -105,8 +87,3 @@ microk8s config > ~/.kube/config
```
**Note:** This must be re-applied after MicroK8s upgrades or cert rotations, as MicroK8s may regenerate the CA without the extension. The fix needs to run on every node in the cluster.
Checked 2026-10-25: the CA from this fix (`CN=microk8s-ca`, 2026-03-21, valid
until 2036) still has the extension after the refreshes to 1.34, 1.35 and
1.36. Check with
`openssl x509 -in /var/snap/microk8s/current/certs/ca.crt -noout -ext keyUsage`.
-4
View File
@@ -8,10 +8,6 @@ spec:
# start an in-place major upgrade. CNPG filled this in at creation.
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
primaryUpdateStrategy: unsupervised
resources:
requests:
cpu: 50m
memory: 256Mi
bootstrap:
initdb:
database: authentik
-14
View File
@@ -1,19 +1,5 @@
# Velero Backup Testing Guide
**Use `restore-test.sh`.** It restores a few namespaces from a real backup
into a throwaway k3s in Docker, from a read-only copy of the bucket, and
checks that every volume comes back with the bytes it was backed up with:
```bash
backup-testing/restore-test.sh # newest daily backup; ntfy, freshrss, actual-budget
backup-testing/restore-test.sh <backup> <namespace>... # something else
```
How it works and why it copies instead of pointing Velero at the NAS is at
the top of the script. The rest of this README describes the older k3d
scripts, which predate Envoy Gateway and still expect `*.hensen.io` and
sol-auth.
This guide explains how to test your Velero backups using a temporary K3d cluster.
## Features
-222
View File
@@ -1,222 +0,0 @@
#!/bin/bash
# Restore a few namespaces from the real Velero backups into a throwaway
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
# changes:
# - one backup (backups/<name>/) and the kopia repositories of the chosen
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with
# rclone into a Docker volume. A second Velero on the real bucket could
# run kopia maintenance on the same repositories as the cluster's own
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
# ReadOnly backup storage location
# - storage class longhorn becomes local-path; only core workload
# resources are restored (no routes, certificates, ExternalSecrets or
# NetworkPolicies)
# - every pod volume restore has to complete with as many bytes as its
# backup had, and the restored deployments have to become available
#
# Usage (on a machine with docker, kubectl and jq):
# backup-testing/restore-test.sh [backup] [namespace ...]
# Defaults: the newest Completed daily-backup, and ntfy freshrss
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
#
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
# into a temporary directory, and the copied backup holds every Secret of
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
# cluster (kubeconfig in the printed directory) until you run it with
# CLEANUP=1. FROM_VOLUME=<docker volume> takes the bucket from a volume
# instead of the NAS (for rehearsals; laid out as <bucket>/backups/...).
#
# Rehearsed on 2026-10-25 against a backup made in the test cluster itself,
# which is how two pitfalls showed up: Velero skips hostPath volumes, and
# k3s only provisions local PVs (which it can back up and restore) for a
# storage class with the annotation defaultVolumeType: local.
set -euo pipefail
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
PORT=${PORT:-16443}
K3S=rancher/k3s:v1.34.9-k3s1
RCLONE=rclone/rclone:1.75.1
HELM=alpine/helm:3.19.0
VELERO_CHART=12.2.1
AWS_PLUGIN=velero/velero-plugin-for-aws:v1.14.4
NET=restore-test VOL=restore-test-s3
cleanup() {
docker rm -f -v restore-k3s restore-s3 >/dev/null 2>&1 || true
docker volume rm "$VOL" >/dev/null 2>&1 || true
docker network rm "$NET" >/dev/null 2>&1 || true
[[ -n ${WORK:-} ]] && rm -rf "$WORK"
}
if [[ ${CLEANUP:-} == 1 ]]; then
cleanup; rm -rf "${TMPDIR:-/tmp}"/restore-test.*; echo "cleaned up"; exit 0
fi
if docker ps -a --format '{{.Names}}' | grep -qx restore-k3s; then
echo "A test cluster from an earlier run exists; run with CLEANUP=1 first." >&2
exit 1
fi
WORK=$(mktemp -d "${TMPDIR:-/tmp}/restore-test.XXXXXX")
chmod 700 "$WORK"
[[ ${KEEP:-} == 1 ]] || trap cleanup EXIT
pk() { kubectl --kubeconfig "$PROD_KUBECONFIG" "$@"; }
tk() { kubectl --kubeconfig "$WORK/kubeconfig" "$@"; }
# wait_for <seconds> <what> <command...>
wait_for() {
local limit=$1 what=$2; shift 2
local end=$((SECONDS + limit))
until "$@" >/dev/null 2>&1; do
if ((SECONDS > end)); then echo "timed out waiting for $what" >&2; exit 1; fi
sleep 5
done
}
# Endpoint, region and bucket as Velero itself uses them. MinIO answers a
# request signed for the wrong region (rclone defaults to us-east-1) with a
# bare 400 on HEAD.
if [[ -n ${FROM_VOLUME:-} ]]; then
BUCKET=${BUCKET:-velero-backups}
else
bsl=$(pk -n velero get backupstoragelocations default -o json)
MINIO=${MINIO:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
fi
backup=${1:-}
shift || true
namespaces=("$@")
[[ ${#namespaces[@]} -gt 0 ]] || namespaces=(ntfy freshrss actual-budget)
if [[ -z $backup ]]; then
backup=$(pk -n velero get backups.velero.io -o json | jq -r '[.items[]
| select(.metadata.labels["velero.io/schedule-name"] == "daily-backup" and .status.phase == "Completed")]
| max_by(.status.startTimestamp).metadata.name')
fi
echo "== backup $backup, namespaces: ${namespaces[*]}"
docker network create "$NET" >/dev/null
docker volume create "$VOL" >/dev/null
if [[ -n ${FROM_VOLUME:-} ]]; then
echo "== copy from the volume $FROM_VOLUME"
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
else
echo "== copy from the NAS (read-only)"
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$MINIO" "$REGION" >> "$WORK/nas.env")
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
copy "backups/$backup"
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
rm -f "$WORK/nas.env"
fi
echo "== local S3 and k3s"
docker run -d --name restore-s3 --network "$NET" -v "$VOL:/data" "$RCLONE" \
serve s3 /data --addr :8080 --auth-key test,restore-test-secret >/dev/null
s3ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" restore-s3)
docker run -d --name restore-k3s --privileged --network "$NET" -p "127.0.0.1:$PORT:6443" "$K3S" \
server --disable traefik --disable metrics-server --tls-san 127.0.0.1 --tls-san restore-k3s >/dev/null
node_ready() { docker exec restore-k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; }
wait_for 180 "k3s" node_ready
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$PORT#" > "$WORK/kubeconfig"
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed 's#https://127.0.0.1:6443#https://restore-k3s:6443#' > "$WORK/kubeconfig-net"
echo "== Velero"
cat > "$WORK/values.yaml" <<EOF
configuration:
backupStorageLocation:
- name: default
provider: aws
bucket: $BUCKET
accessMode: ReadOnly
config:
region: minio
s3ForcePathStyle: "true"
s3Url: http://$s3ip:8080
volumeSnapshotLocation: []
defaultRepoMaintainFrequency: 8760h
credentials:
secretContents:
cloud: |
[default]
aws_access_key_id=test
aws_secret_access_key=restore-test-secret
initContainers:
- name: velero-plugin-for-aws
image: $AWS_PLUGIN
volumeMounts:
- mountPath: /target
name: plugins
deployNodeAgent: true
snapshotsEnabled: false
upgradeCRDs: false
EOF
docker run --rm --network "$NET" -e HOME=/tmp -e KC="$(base64 < "$WORK/kubeconfig-net")" \
-e VALUES="$(base64 < "$WORK/values.yaml")" --entrypoint sh "$HELM" -c '
echo "$KC" | base64 -d > /tmp/kc && echo "$VALUES" | base64 -d > /tmp/v.yaml &&
helm --kubeconfig /tmp/kc install velero velero --repo https://vmware-tanzu.github.io/helm-charts \
--version '"$VELERO_CHART"' -n velero --create-namespace -f /tmp/v.yaml >/dev/null'
tk apply -f - >/dev/null <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: change-storage-class
namespace: velero
labels:
velero.io/plugin-config: ""
velero.io/change-storage-class: RestoreItemAction
data:
longhorn: local-path
EOF
wait_for 300 "Velero to sync $backup" tk -n velero get backups.velero.io "$backup"
echo " Velero sees $backup"
echo "== restore"
tk apply -f - >/dev/null <<EOF
apiVersion: velero.io/v1
kind: Restore
metadata:
name: restore-test
namespace: velero
spec:
backupName: $backup
includedNamespaces: [$(IFS=,; echo "${namespaces[*]}")]
# PersistentVolumes have to be included: only then does Velero clear the
# PVC's volumeName. It doesn't restore a PV whose data comes from kopia;
# k3s provisions a new one.
includedResources: [namespaces, persistentvolumeclaims, persistentvolumes, pods, deployments,
replicasets, statefulsets, services, configmaps, secrets, serviceaccounts]
restorePVs: true
EOF
restore_done() { [[ $(tk -n velero get restore restore-test -o jsonpath='{.status.phase}') =~ ^(Completed|PartiallyFailed|Failed|FailedValidation)$ ]]; }
wait_for 1800 "the restore" restore_done
phase=$(tk -n velero get restore restore-test -o jsonpath='{.status.phase}')
echo " restore: $phase"
echo "== check"
failed=0
expected=$(docker run --rm -v "$VOL:/data" alpine:3.22 zcat "/data/$BUCKET/backups/$backup/$backup-podvolumebackups.json.gz" |
jq -r --arg ns "$(IFS=,; echo "${namespaces[*]}")" '.[] | select(.spec.pod.namespace as $p | $ns | split(",") | index($p))
| "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0)"' | sort -u)
restored=$(tk -n velero get podvolumerestores.velero.io -o json |
jq -r '.items[] | "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0) \(.status.phase)"' | sort -u)
while read -r vol bytes; do
[[ -z $vol ]] && continue
got=$(awk -v v="$vol" '$1 == v {print $2, $3}' <<<"$restored")
if [[ $got == "$bytes Completed" ]]; then
echo " ok $vol ($bytes bytes)"
else
echo " FAILED $vol: backed up $bytes bytes, restore: ${got:-none}"; failed=1
fi
done <<<"$expected"
for ns in "${namespaces[@]}"; do
if tk -n "$ns" wait --for=condition=Available deployment --all --timeout=300s >/dev/null 2>&1; then
echo " ok $ns: deployments available"
else
echo " FAILED $ns: deployments not available"; tk -n "$ns" get pods; failed=1
fi
done
[[ $phase == Completed ]] || failed=1
[[ ${KEEP:-} == 1 ]] && echo "Kept; kubeconfig: $WORK/kubeconfig. Remove with CLEANUP=1 $0"
if [[ $failed == 0 ]]; then echo "== restore test passed"; else echo "== restore test FAILED"; exit 1; fi
+9 -7
View File
@@ -123,13 +123,15 @@ D. auth.scoutingmbg.nl (2026-10-25)
## Open points
- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
all the records it moved, and node001 tunnels all four home. ns1 lagged
behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
`a` now means node001, which is fine: the WordPress site sends no mail
and the group's mail goes through Google (`mx`)
- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
foto.scoutingmbg.nl from a CNAME to `mbg.hensen.io` (the home IP) to a
CNAME to `rubenhensen.nl`, which is node001 on Hetzner. Only ns2 has it
(serial 2026102471); ns1 is stuck on serial 2026060640 from June and
still answers `mbg.hensen.io`. node001 now tunnels foto.scoutingmbg.nl to
the home cluster like its other hosts, so both answers work. Keep
hensen.io until ns1 has caught up; ask the DNS host to fix ns1, and
decide whether foto should stay behind node001 or point straight home
(CNAME `scoutingmbg.nl`)
- Public shared links become public for real once sol-auth is gone.
- Photos leiding back up from their phones stay private until they add
them to a shared album.
+2 -13
View File
@@ -38,15 +38,14 @@ between them live.
## Before: shrink what has to move
- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
- [ ] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
snapshots from before October 2026, nearly all on immich-claim
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
don't depend on them. Deleting a snapshot merges it into the next one,
which writes a lot, so do this in a quiet hour, one volume at a time.
Happened on 2026-10-24, from a Claude Code session at Ruben's request
(the 45 snapshots from before October 2026, small volumes first): immich-claim's
Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
old snapshots were removed at 14:43 UTC and Longhorn merged them one
after another, about 45 minutes each, until 23:15. For those 8.5 hours
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
@@ -55,16 +54,6 @@ between them live.
on immich-claim is one 233 GiB snapshot (`system-backup-0b7f8c6047294b36`,
from the Longhorn system backup of 2026-10-22) that everything merged
into; 148 snapshots hold 273 GiB across all volumes.
- [ ] Delete the snapshots the Longhorn system backups left behind: 116
`system-backup-*` snapshots (one per volume for each system backup
before an upgrade step on 2026-10-22 and 23) hold 258 GiB. The system
backups on the NAS don't need them. The small volumes are cheap; on
immich-claim the 233 GiB one is the base the rest merged into, so
deleting it means hours of merging like on 2026-10-24: one night, or
after the move to the NVMe drive. Then trim (next item); deleted files
only free space once both are done. Next time, take system backups with
the volume backup policy `disabled` when the nightly backups are fresh,
so they create no snapshots
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
so blocks of deleted files are released.
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
+28 -214
View File
@@ -8,15 +8,11 @@ bottom.
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
and s2 (home) watch each other and alert through ntfy; s2 also gets an
hourly report from inside the cluster on the backups, Vault's seal, the
Argo CD apps and the CNPG clusters.
Every app namespace has NetworkPolicies, and CI renders and validates all
apps on every PR. node001 runs mail and tunnels the public hosts home.
35 Argo CD apps, all Synced/Healthy; every Longhorn volume has one
replica. Backups go to the NAS (192.168.1.99): Longhorn to NFS at 00:00
(7 kept), Velero to MinIO at 01:00 (daily kept 30 days, weekly 90 days).
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
through ntfy. node001 runs mail and tunnels the public hosts home.
## Follow-ups from this week
@@ -34,9 +30,8 @@ docs/longhorn-second-ssd.md:
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
(`backend-*.tar`, `dqlite-pre-*`)
- [ ] Log in to immich as a jeugdlid (quota 0)
- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
2026-10-25). The snap is held, so microk8s gets no updates at all until
someone refreshes it
- [ ] Back to 1.36/stable once it is at v1.36.5 or later. The snap is held,
so microk8s gets no updates at all until someone refreshes it
- [ ] Second SSD, docs/longhorn-second-ssd.md
- [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
@@ -50,9 +45,8 @@ group's ns2: ns1 is stuck on an old serial and still answers
mbg.hensen.io (docs/immich-scouts-online.md).
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
`rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
cluster uses hensen.io anymore. hensen.io can expire
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
rubenhensen.nl too
### 2. Off-site backups
@@ -83,34 +77,7 @@ friend.
the NAS
- [ ] Decide which copy of the immich library goes off-site (see 5)
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
Planned on 2026-10-25, not run: a second Velero on the real bucket is
risky even with a ReadOnly location, because kopia could then run
maintenance on the same repositories from two clusters. Instead: copy
one backup (`backups/<name>/`) and the kopia repositories of a few small
namespaces (`kopia/{gatus,ntfy,freshrss}/`) read-only from the NAS with
rclone into a local folder, serve that with `rclone serve s3`, and
restore into a throwaway k3s in Docker (Velero 1.18.2, node agent on,
storage class longhorn mapped to local-path, without httproutes,
certificates, referencegrants and networkpolicies). It needs the MinIO
key outside the cluster, which wants your go-ahead; better still, a
MinIO user with read-only access to `velero-backups`.
Built as `backup-testing/restore-test.sh` (2026-10-25) and rehearsed
end to end against a backup made inside the test cluster, with a
ReadOnly location: data restored byte for byte (same sha256), deployment
available, everything removed afterwards. Two pitfalls found on the way
are in the script's header. Not yet run against the real backups:
- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
volume came back with its backed-up byte count (actual-budget 8.4 MB,
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
MinIO runs region `minio`; the script now reads endpoint, region and
bucket from Velero's location. Copying freshrss's kopia repository took
14 of the ~17 minutes
- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
the cluster itself: the Mac (launchd) or node001, ideally with a
read-only MinIO user
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth
### 3. authentik 2026.2.1 → 2026.8.5
@@ -138,13 +105,10 @@ April and May.
- [x] Log in once through each app: immich through Scouts Online, Argo CD,
Vault, Longhorn, FreshRSS, Actual, mail
- [ ] After a week without problems: delete the dump and the snapshot
- [x] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`), required
from 2026.11: already `https://authentik.rubenhensen.nl`, filled in by
the 2026.8 migration from the embedded outpost's host. In 2026.8.5 it is
only stored; on main it fills itself in the same way and sets the
embedded outpost's host from it. OIDC issuers still come from the
request's host, so immich's issuer on auth.scoutingmbg.nl is unaffected
(checked 2026-10-25 in the source)
- [ ] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`) becomes
required in 2026.11. authentik answers on authentik.rubenhensen.nl and
auth.scoutingmbg.nl (immich's issuer), so find out first what the
setting does with two hostnames
### 4. Vault 1.17.2 → 2.2.1
@@ -165,8 +129,8 @@ unseals it.
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
doesn't exist, so the OIDC mount's listing visibility was never set.
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
- [x] There is one unseal key (1 share, threshold 1), kept safe outside
this cluster (2026-10-25); without it the data in Vault is lost
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
places outside this cluster; without it the data in Vault is lost
- [x] Unsealing is in the SSD install steps
- [ ] After a week without problems: delete the three snapshots
@@ -215,21 +179,15 @@ until 2026-10-21 without anyone noticing.
- [x] A stopgap for the worst blind spots (2026-10-25,
`gatus/backup-check.yaml`): an hourly CronJob reads the API and pushes
six external endpoints to s2 (group "cluster"): the newest daily and
four external endpoints to s2 (group "cluster"): the newest daily and
weekly Velero backup Completed and recent, every attached Longhorn
volume healthy with a backup younger than 26h, no Vault pod sealed,
every Argo CD app Synced and Healthy, every CNPG cluster healthy.
volume healthy with a backup younger than 26h, and no Vault pod sealed.
Gatus alerts through ntfy after two failed reports, or when no report
arrives for 3 hours. The token comes from an ESO Password generator.
Tested with made-up data for each failure (a Failed or old backup, a
degraded or unbacked-up volume, a sealed pod)
- [x] metrics-server (2026-10-25, `apps/templates/metrics-server-helm.yaml`):
`kubectl top` works; it keeps its data in memory, so it adds no disk
writes. The first look: immich v3 checks file checksums every night from
03:00 UTC for up to an hour (IntegrityService), reading ~100 MB/s from
sda; it doesn't overlap the backups at 00:00 and 01:00
- [ ] After the second SSD (a time series database writes all the time
and would add to today's stalls): Prometheus or
and would add to today's stalls): metrics-server, then Prometheus or
VictoriaMetrics with Grafana, alerting to ntfy. Loki later
- [ ] Alerts worth having: disk latency and fill, lease losses and
controller restarts, backups that don't complete, Longhorn volumes that
@@ -277,48 +235,9 @@ Counted 2026-10-25 over the 73 running containers:
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
default) still have none; `default` can reach the LAN and everything
without a policy
- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
had none except the Vault server, from `kubectl top` every 5 minutes
from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
floor). Rolled out in about eight minutes; both CNPG instances
restarted and came back healthy, the OIDC endpoints, the logins and
immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
requested. Still without requests: Longhorn (22 containers, its chart),
the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
kube-system
- [ ] Memory limits, once there are daytime numbers: the night ones miss
immich's machine learning at work (it idles at 234Mi)
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
2026-10-25, its third restart: a crawler asked for 88 pages and assets
within a few seconds (normally ~6 a minute), Apache started a PHP worker
for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
the container went past its 1Gi limit. Unrelated to the
NetworkPolicies. Now capped at 12 workers through the chart's
`apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
with `apache2ctl -t` in the same image first. Replaying the burst (90
page requests, 40 at a time): all 200, at most 13 Apache processes and
572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
peak) instead of a crash
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
cluster between Envoy and WordPress that keeps anonymous GETs for a few
minutes and passes anything with a WordPress cookie through. A page
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
waiting in a burst is queueing; the cap alone already prevents the crash
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
little here, `wordpress-importer` is usually for a one-off import, and
`code-snippets` runs PHP from the dashboard
- [ ] Rate limiting per visitor in Envoy needs the client's address, which
node001 loses: it passes TLS through, so every request arrives from
46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
(ClientTrafficPolicy `enableProxyProtocol`)
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
SSD's stalls. Still without probes: authentik's LDAP outpost (deployed
by authentik), the ESO controller, dex and Argo CD's redis (upstream
manifests)
- [ ] Requests and memory limits, the big ones first (immich, authentik,
Argo CD's repo-server)
- [ ] Probes
- [ ] Non-root where the image allows it
### 8. Renovate and CI
@@ -326,64 +245,9 @@ Counted 2026-10-25 over the 73 running containers:
10 Renovate PRs are open, the oldest from February, and nothing checks a
PR before Argo CD applies it.
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
does (the apps chart, every Application's chart with its values, value
files and parameters, kustomizations, plain folders, and
external-snapshotter cloned with its patches) and validates the result
with kubeconform (strict, CRD schemas from the datree catalog), plus a
duplicate-key check on every YAML file. Helm 4.2.1 like Argo CD 3.5,
versions pinned and checksummed, kept current by Renovate through the
`# renovate:` comments. Tested by planting a typo in a Deployment, a
chart version that doesn't exist and a value the immich chart's schema
rejects; all three fail the run. Locally:
`docker run --rm -v $PWD:/repo:ro <image with helm, kustomize, kubeconform, PyYAML> python /repo/tools/ci/validate.py`
- It found `syncPolicy.automated` twice in
`apps/templates/snapshot-controller-helm.yaml` (once with `prune: true`,
added 2025-03-03); Argo CD kept the last one, without prune. Now listed
once, as it ran. Prune was probably meant; turn it on deliberately if so
- [x] This repo's own rules, in `tools/ci/validate.py` rather than
conftest (2026-10-25), on what the repo's own folders render: no Secret
with data in git, and every image has a tag other than latest (both
fail the run; tested with a planted Secret and an untagged image), plus
PR annotations for containers without requests or probes. freshrss's
`alpine/git` init container had no tag; now `v2.52.0`
- [ ] Automerge patch versions of low-risk apps now that CI runs
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
`argocd` manager and a custom `github-tags` regex both matched. The
regex is gone; Renovate should close #15 by itself
- [x] Renovate only looked at six folders for images and only at
apps/templates for Applications, so it never offered updates for the
vault, immich, velero, mariadb-operator and actualbudget charts, or for
images in freshrss, mbgwp, dns, vault and immich, or for CNPG's
`imageName` (authentik still runs PostgreSQL 17.4). Both managers now
cover every app folder, a regex manager reads `imageName` in
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
(those need pg_upgrade or a dump). Expect a batch of new PRs
(2026-10-25). They only come once fewer than 10 Renovate PRs are open
(`config:recommended`'s `prConcurrentLimit`); until then they wait
under "Rate-Limited" on the Dependency Dashboard (#2)
- The other folders were left out with a `(?!…)` lookahead. Renovate
compiles these patterns with RE2, which has no lookahead, so it
rejected the config and stopped opening PRs (#32). Each manager now
takes every folder and skips the others through its own
`ignorePaths`. Renovate 44.181.6 with RE2 loaded fails the old config
with #32's error and passes the new one, and its matcher picks the
same 166 files as the old pattern did
- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
v1.21.3 (security release; all three pods on v1.21.3), #24
external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
its own beforehand, the nightly ones are the way back)
- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
manual rebase (#33 had added values on the next line, which git counts
as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
ExternalSecrets refreshed on a force-sync; the 25th is the generated
`gatus/backup-check-token`, which never refreshes by design
- [ ] A GitHub Actions workflow that renders the Helm apps and runs
kubeconform on every PR, with conftest for this repo's rules
- [ ] Automerge patch versions of low-risk apps once that runs
### 9. One node
@@ -399,12 +263,6 @@ step after that, if downtime starts to matter.
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
completed kopia maintenance jobs of repositories that no longer exist
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
volumes over the file paths `data/config.custom.php` and
`data/config-user.custom.php`, so FreshRSS sees two empty directories
there and the volumes hold nothing (0 bytes in every backup). Drop both
mounts and the two PVCs, or make them ConfigMap files if custom config
is wanted
- [ ] `kubectl delete namespace secrets`, empty since its app went
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
@@ -427,53 +285,9 @@ step after that, if downtime starts to matter.
- [ ] opti1, with sudo: `/root/backend-*` and `/root/dqlite-pre-*` (from
before 1.33, 1.34 and the VACUUM; 1.34 migrated the schema, so none of
them fits 1.36) and `~/containerd.bak` (root-owned, from 2025-02)
- [ ] `rm -r ~/cadir` on opti1: a CA key and certificate made four minutes
before the real keyUsage fix on 2026-03-21 (OpenSSL's default subject
"Internet Widgits Pty Ltd", valid for a year) and never installed;
microk8s's CA is the `CN=microk8s-ca` one from 19:16 that day, valid
until 2036
### 11. Known vulnerabilities
A one-off scan on 2026-10-25 of all 52 images running in the cluster
(Trivy 0.75.0, remote, linux/amd64, HIGH and CRITICAL only). Counts are
critical / high:
| Image | Crit | High | What to do |
|---|---|---|---|
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; goes with the cleanup from 2026-10-27 |
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. Renovate now offers 17.7; a `17.x-standard-trixie` image is more current |
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
| coredns/coredns:1.10.1 | 5 | 70 | see below |
| dexidp/dex:v2.45.1, argocd v3.5.6, redis 8.2.3 | 4, 1, 2 | | come with Argo CD |
| mariadb:11.4.5 | 4 | 65 | the newest 11.4 patch; Renovate now covers it |
| calico v3.29.3 | 2-3 | ~50 | comes with microk8s |
| goauthentik ldap / server 2026.8.5 | 3, 1 | 74, 65 | the next authentik patch |
Nothing critical in Envoy, Vault, Velero, cert-manager, ESO, ntfy, gatus or
metrics-server.
- [ ] 16 volumes still have their engine in Longhorn 1.7.3's instance
manager (`instance-manager-07d66a5…`, running since March): vault ×3,
authentik-postgres, immich-postgres (PG14), mbgwp (mariadb and
wordpress), gitea, freshrss ×3, ntfy, gatus, serpbear, bolletjestrui,
actual-budget. On one node a pod restart never detaches the volume (the
new pod lands on the same node), so only a real detach moves the engine
to 1.13's instance manager: scale to 0, wait until Longhorn shows the
volume detached, scale back (Argo CD's selfHeal undoes a scale-down, so
pause it for the app). Vault comes back sealed; CNPG clusters want
`cnpg.io/hibernation=on` instead of a scale-down. The old instance
manager goes away by itself once it serves nothing
- [ ] CoreDNS runs 1.10.1 from when the addon was enabled; the addon
repository has 1.14.3. `sudo microk8s disable dns && sudo microk8s enable dns:1.1.1.1`
(a short DNS outage for the whole cluster; check the Corefile after)
- [ ] The Trivy Operator for scans that keep running (on the wishlist)
- [ ] `~/cadir` on opti1 holds a CA key and certificate from 2026-03-21
that are not microk8s's current CA. Find out whether anything still
trusts it; if not, delete it
## wishlist.txt (2026-01-25)
-7
View File
@@ -282,13 +282,6 @@ whatever time of day the hourly schedule had drifted to.
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
pressure 78%. The CSI sidecars and Longhorn's global manager lost
their leases at 01:31
Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
11:30, leaving out the CNPG range cancels): all 67 minutes with errors
had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
"full" pressure at 45-78%, against a median minute peak of 24 ms. The
worst minute after the merges was the rollout of eight merged PRs at
2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
CNPG and Envoy Gateway held their 60s leases through all of it
Ubuntu's sysstat already keeps per-device history every 10 minutes
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
+1 -18
View File
@@ -23,7 +23,7 @@ spec:
io.kompose.service: freshrss
spec:
initContainers:
- image: alpine/git:v2.52.0
- image: alpine/git
name: git
command:
- git
@@ -79,23 +79,6 @@ spec:
ports:
- containerPort: 80
protocol: TCP
resources:
requests:
cpu: 50m
memory: 176Mi
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
# browser.
readinessProbe:
tcpSocket:
port: 80
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 80
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
volumeMounts:
- mountPath: /var/www/FreshRSS/data
name: data
+1 -29
View File
@@ -11,8 +11,6 @@
# younger than 26h (detached volumes are skipped; the recurring job skips
# them too)
# - vault-unsealed: all Vault pods report vault-sealed=false
# - argocd-apps: every Argo CD application is Synced and Healthy
# - postgres: every CloudNativePG cluster is in a healthy state
#
# The token is generated by ESO and never leaves the cluster; Gatus reads
# it from the same Secret.
@@ -65,12 +63,6 @@ rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["list"]
- apiGroups: ["argoproj.io"]
resources: ["applications"]
verbs: ["list"]
- apiGroups: ["postgresql.cnpg.io"]
resources: ["clusters"]
verbs: ["list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
@@ -178,24 +170,6 @@ data:
return f"sealed: {', '.join(sealed)}" if sealed else None
def argocd():
bad = []
for a in items("/apis/argoproj.io/v1alpha1/namespaces/argocd/applications"):
s = a.get("status", {})
sync = s.get("sync", {}).get("status")
health = s.get("health", {}).get("status")
if sync != "Synced" or health != "Healthy":
bad.append(f"{a['metadata']['name']} {sync}/{health}")
return "; ".join(bad) or None
def postgres():
bad = [f"{c['metadata']['namespace']}/{c['metadata']['name']}: {c.get('status', {}).get('phase')}"
for c in items("/apis/postgresql.cnpg.io/v1/clusters")
if c.get("status", {}).get("phase") != "Cluster in healthy state"]
return "; ".join(bad) or None
def push(key, error):
query = {"success": "false" if error else "true"}
if error:
@@ -211,8 +185,6 @@ data:
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
"cluster_longhorn": lambda: longhorn(26),
"cluster_vault-unsealed": vault,
"cluster_argocd-apps": argocd,
"cluster_postgres": postgres,
}
failed_pushes = 0
for key, check in checks.items():
@@ -256,7 +228,7 @@ spec:
type: RuntimeDefault
containers:
- name: check
image: python:3.15.1-alpine3.24
image: python:3.13.16-alpine3.24
command: ["python", "/script/check.py"]
env:
- name: PYTHONDONTWRITEBYTECODE
-12
View File
@@ -169,15 +169,3 @@ external-endpoints:
heartbeat:
interval: 3h
alerts: *cluster-alert
- name: argocd-apps
group: cluster
token: ${BACKUP_CHECK_TOKEN}
heartbeat:
interval: 3h
alerts: *cluster-alert
- name: postgres
group: cluster
token: ${BACKUP_CHECK_TOKEN}
heartbeat:
interval: 3h
alerts: *cluster-alert
+1 -20
View File
@@ -20,7 +20,7 @@ spec:
fsGroup: 1000
containers:
- name: gitea
image: gitea/gitea:28.2.0
image: gitea/gitea:1.27.3
env:
- name: USER_UID
value: "1000"
@@ -50,25 +50,6 @@ spec:
- containerPort: 3000
name: http
protocol: TCP
resources:
requests:
cpu: 20m
memory: 336Mi
# /api/healthz checks the database and the cache.
readinessProbe:
httpGet:
path: /api/healthz
port: http
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /api/healthz
port: http
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
volumeMounts:
- mountPath: /data
name: data
-14
View File
@@ -63,10 +63,6 @@ spec:
main:
containers:
main:
resources:
requests:
cpu: 10m
memory: 64Mi
image:
repository: docker.io/valkey/valkey
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
@@ -89,10 +85,6 @@ spec:
image:
repository: ghcr.io/immich-app/immich-server
pullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 1280Mi
# Node trusts its built-in roots, which still include the
# ones above the compromised eMudhra intermediates. Use
# patched-ca-bundle through OpenSSL until Node ships a fix.
@@ -117,12 +109,6 @@ spec:
image:
repository: ghcr.io/immich-app/immich-machine-learning
pullPolicy: IfNotPresent
# Idle at night (234Mi); the models take over a GiB when
# it works.
resources:
requests:
cpu: 50m
memory: 1Gi
env:
TRANSFORMERS_CACHE: /cache
HF_XET_CACHE: /cache/huggingface-xet
-4
View File
@@ -9,10 +9,6 @@ spec:
# CNPG image with VectorChord and pgvector, which immich 2.x and 3.x use
# for its vector indexes (pgvecto.rs is gone in immich 3.0).
imageName: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
resources:
requests:
cpu: 20m
memory: 320Mi
postgresql:
shared_preload_libraries:
-12
View File
@@ -16,26 +16,14 @@ spec:
targetRevision: 25.10.4
helm:
values: |
resources:
requests:
cpu: 10m
memory: 48Mi
metrics:
enabled: true
serviceMonitor:
enabled: false
webhook:
resources:
requests:
cpu: 10m
memory: 32Mi
serviceMonitor:
enabled: false
certController:
resources:
requests:
cpu: 10m
memory: 32Mi
serviceMonitor:
enabled: false
syncPolicy:
-35
View File
@@ -1,35 +0,0 @@
# Garage on the NAS
S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
the cluster. Replaced MinIO, which is no longer maintained. One node, data
in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
its PATH.
## Setup
`garage.toml` here has placeholders for the two secrets; on the NAS they
were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
`openssl rand -base64 32` (`admin_token`), and only exist there.
```
cd /volume1/docker/garage
docker compose up -d
g="docker exec garage /garage"
$g layout assign -z nas -c 800G <node id from $g status>
$g layout apply --version 1
$g bucket create velero-backups
$g key create velero
$g bucket allow --read --write --key velero velero-backups
```
Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
`secret_access_key`) and reaches Velero through
`velero/external-secret.yaml`. Without the cluster, `$g key info velero
--show-secret` on the NAS shows it again.
Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
after merging, `docker compose pull && docker compose up -d` on the NAS.
-12
View File
@@ -1,12 +0,0 @@
services:
garage:
image: dxflrs/garage:v2.5.0
container_name: garage
restart: unless-stopped
ports:
- "3900:3900" # S3 API
- "3903:3903" # admin API
volumes:
- ./garage.toml:/etc/garage.toml:ro
- ./meta:/var/lib/garage/meta
- ./data:/var/lib/garage/data
-29
View File
@@ -1,29 +0,0 @@
# One node on the Synology. Velero's kopia data is encrypted, so
# compression would only cost CPU.
#
# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
# shutdown, and a single node has no other node to repair from. With
# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
# with `garage convert-db`.
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
metadata_fsync = true
data_fsync = true
metadata_auto_snapshot_interval = "6h"
replication_factor = 1
compression_level = "none"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "@RPC_SECRET@"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "@ADMIN_TOKEN@"
+1 -1
View File
@@ -16,7 +16,7 @@ spec:
spec:
containers:
- name: ntfy
image: binwiederhier/ntfy:v2.34.0
image: binwiederhier/ntfy:v2.28.0
args: ["serve"]
env:
- name: NTFY_BASE_URL
+17 -43
View File
@@ -5,16 +5,8 @@
"customManagers:githubActionsVersions"
],
"argocd": {
"description": "Every app folder; Renovate's regexes are RE2, which has no lookahead, so the other folders are skipped through ignorePaths",
"managerFilePatterns": [
"/^[^/]+/.+\\.yaml$/"
],
"ignorePaths": [
"nix-infra-machine/**",
"backup-testing/**",
"docs/**",
"tools/**",
".github/**"
"/apps/templates/.+\\.yaml$/"
]
},
"helm-values": {
@@ -23,16 +15,13 @@
]
},
"kubernetes": {
"description": "Every app folder, as for argocd",
"managerFilePatterns": [
"/^[^/]+/.+\\.yaml$/"
],
"ignorePaths": [
"nix-infra-machine/**",
"backup-testing/**",
"docs/**",
"tools/**",
".github/**"
"/apps/templates/.+\\.yaml$/",
"/serpbear/.+\\.yaml$/",
"/bolletjestrui/.+\\.yaml$/",
"/gitea/.+\\.yaml$/",
"/gatus/.+\\.yaml$/",
"/ntfy/.+\\.yaml$/"
]
},
"customManagers": [
@@ -42,20 +31,19 @@
"/apps/templates/.+\\.yaml$/"
],
"matchStrings": [
"repoURL: (?<registryUrl>.*?)\\n\\s+targetRevision: (?<currentValue>[^\\s]+)\\n\\s+chart: (?<depName>[^\\s]+)"
"repoURL: https://github.com/(?<depName>[^/]+/[^/\\.]+)\\.git\\n\\s+targetRevision: (?<currentValue>.*)"
],
"datasourceTemplate": "helm"
"datasourceTemplate": "github-tags"
},
{
"customType": "regex",
"description": "CloudNativePG clusters name their image in imageName",
"managerFilePatterns": [
"/^[^/]+/postgresql-cluster[^/]*\\.yaml$/"
"/apps/templates/.+\\.yaml$/"
],
"matchStrings": [
"imageName: (?<depName>[^:\\s]+):(?<currentValue>\\S+)"
"repoURL: (?<registryUrl>.*?)\\n\\s+targetRevision: (?<currentValue>[^\\s]+)\\n\\s+chart: (?<depName>[^\\s]+)"
],
"datasourceTemplate": "docker"
"datasourceTemplate": "helm"
}
],
"packageRules": [
@@ -68,26 +56,12 @@
"separateMinorPatch": true
},
{
"description": "CI renders with the same Helm as Argo CD (3.5 ships Helm 4.2); move it together with Argo CD",
"matchFileNames": [
".github/workflows/**"
"matchDatasources": [
"github-tags"
],
"matchDepNames": [
"helm/helm"
],
"enabled": false
},
{
"description": "A PostgreSQL major upgrade needs pg_upgrade or a dump, not a tag bump",
"matchDepNames": [
"ghcr.io/cloudnative-pg/postgresql",
"ghcr.io/tensorchord/cloudnative-vectorchord",
"ghcr.io/tensorchord/cloudnative-pgvecto.rs"
],
"matchUpdateTypes": [
"major"
],
"enabled": false
"semanticCommits": "enabled",
"commitMessageTopic": "{{depName}}",
"separateMinorPatch": true
}
]
}
-17
View File
@@ -56,23 +56,6 @@ spec:
ports:
- containerPort: 3000
protocol: TCP
resources:
requests:
cpu: 10m
memory: 160Mi
readinessProbe:
httpGet:
path: /login
port: 3000
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
tcpSocket:
port: 3000
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
volumeMounts:
- mountPath: /app/data
name: data
+3 -57
View File
@@ -15,13 +15,6 @@ in the app folders. Sources:
same, with the Application's kustomize patches applied
Every YAML file in the repo is also checked for duplicate keys, which
YAML parsers resolve silently by keeping the last one.
Repo rules, on what this repo's own folders render (not on charts or
remote bases):
- no Secret with data: secrets come from Vault through ESO
- every image has a tag other than latest
- warnings (annotations on the PR) for containers without requests or
probes
"""
import fnmatch
import os
@@ -29,7 +22,6 @@ import pathlib
import subprocess
import sys
import tempfile
import time
import yaml
@@ -46,15 +38,7 @@ failures = []
def run(cmd, **kw):
# Registries rate-limit now and then (ghcr.io answers 429); try again.
for attempt in range(3):
try:
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
except subprocess.CalledProcessError as e:
transient = any(s in (e.stderr or "") for s in ("429", "toomanyrequests", "timeout", "connection reset"))
if not transient or attempt == 2:
raise
time.sleep(10 * (attempt + 1))
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
class UniqueKeyLoader(yaml.SafeLoader):
@@ -94,41 +78,7 @@ def check_duplicate_keys():
failures.append(f"{path.relative_to(ROOT)}: {e}")
def pod_specs(doc):
spec = doc.get("spec") or {}
kind = doc.get("kind")
if kind == "Pod":
yield spec
elif kind == "CronJob":
yield spec.get("jobTemplate", {}).get("spec", {}).get("template", {}).get("spec", {})
elif kind in ("Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"):
yield spec.get("template", {}).get("spec", {})
def check_rules(label, docs):
for d in docs:
name = f"{d.get('kind')}/{d.get('metadata', {}).get('name')}"
if d.get("kind") == "Secret" and (d.get("data") or d.get("stringData")):
failures.append(f"{label}: {name} has data in git; put it in Vault and use an ExternalSecret")
for spec in pod_specs(d):
containers = spec.get("containers", []) + spec.get("initContainers", [])
for c in containers:
image = c.get("image", "")
tag = image.split("@")[0].rsplit("/", 1)[-1].partition(":")[2]
if "@" not in image and tag in ("", "latest"):
failures.append(f"{label}: {name} container {c.get('name')} has no fixed tag ({image})")
for c in spec.get("containers", []):
missing = []
if not c.get("resources", {}).get("requests"):
missing.append("requests")
if d.get("kind") in ("Deployment", "StatefulSet", "DaemonSet") and not (
c.get("readinessProbe") or c.get("livenessProbe")):
missing.append("probes")
if missing:
print(f"::warning title={label}::{name} container {c.get('name')} has no {' or '.join(missing)}")
def validate(label, manifests, rules=False):
def validate(label, manifests):
"""kubeconform on rendered manifests; returns the parsed documents."""
try:
docs = [d for d in yaml.safe_load_all(manifests) if d]
@@ -153,8 +103,6 @@ def validate(label, manifests, rules=False):
for l in lines[:-1]:
print(f" {l}")
failures.append(f"{label}: {summary or result.stderr.strip()}")
if rules and docs:
check_rules(label, [d for d in docs if isinstance(d, dict)])
return docs or []
@@ -290,9 +238,7 @@ def main():
failures.append(f"{label}: render failed")
continue
if manifests is not None:
own = src.get("repoURL") in OWN_REPOS and not src.get("chart") and not (
ROOT / src.get("path", ".") / "Chart.yaml").exists()
validate(label, manifests, rules=own)
validate(label, manifests)
if failures:
print(f"\n{len(failures)} problem(s):")
+1 -1
View File
@@ -3,7 +3,7 @@ module dqlite-vacuum
go 1.27.3
require (
github.com/canonical/go-dqlite/v3 v3.0.4
github.com/canonical/go-dqlite/v3 v3.0.1
gopkg.in/yaml.v2 v2.4.0
)
-5
View File
@@ -2,10 +2,7 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -29,9 +26,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-4
View File
@@ -3,10 +3,6 @@ global:
tlsDisable: true
injector:
enabled: true
resources:
requests:
cpu: 10m
memory: 32Mi
# The chart's default anti-affinity forbids two injector pods on one
# node, so on this single node a rolling update can never start the new
# pod while the old one runs. Recreate removes the old pod first, which
-30
View File
@@ -1,30 +0,0 @@
# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
# AWS credentials file format the aws plugin and kopia read.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: velero-s3-credentials
spec:
secretStoreRef:
name: vault-backend
kind: ClusterSecretStore
refreshInterval: 15m
target:
name: velero-s3-credentials
creationPolicy: Owner
template:
engineVersion: v2
data:
cloud: |
[default]
aws_access_key_id={{ .access_key_id }}
aws_secret_access_key={{ .secret_access_key }}
data:
- secretKey: access_key_id
remoteRef:
key: kv/velero-s3
property: access_key_id
- secretKey: secret_access_key
remoteRef:
key: kv/velero-s3
property: secret_access_key
-4
View File
@@ -68,10 +68,6 @@ spec:
nodeAgent:
containerSecurityContext:
privileged: true
resources:
requests:
cpu: 10m
memory: 64Mi
# 512Mi got the server OOMKilled 314 times, including halfway
# through a backup of ~14.7k items.
resources: