Compare commits

..
28 Commits
Author SHA1 Message Date
Ruben Hensen 5bb1422594 Raise two requests after the morning's numbers 2026-10-25 09:43:11 +01:00
Ruben Hensen 72f49bed51 Set resource requests from measured usage 2026-10-25 07:08:16 +01:00
Ruben Hensen f4b42757cd backup-testing: remove k3s's anonymous volumes too 2026-10-25 07:04:17 +01:00
Ruben Hensen f529bd99ee docs: triage PR 31 2026-10-25 07:02:07 +01:00
Ruben Hensen 74dec430b8 docs: the restore test is ready to run 2026-10-25 07:01:04 +01:00
Ruben Hensen 6fe0acc061 backup-testing: a restore test from a read-only copy of the bucket 2026-10-25 07:00:49 +01:00
Ruben Hensen bb34b601d9 docs: the system backups' snapshots hold 258 GiB 2026-10-25 04:35:58 +01:00
Ruben Hensen e98668d5fa docs: the cluster check has six parts 2026-10-25 04:35:02 +01:00
Ruben Hensen 389f911823 gatus: also report unhealthy Argo CD apps and CNPG clusters 2026-10-25 04:34:32 +01:00
Ruben Hensen 8f1c6a5b6e docs: record the repo rules 2026-10-25 04:33:01 +01:00
Ruben Hensen 770631fdc0 ci: check the repo's own manifests for secrets in git and unpinned images 2026-10-25 04:32:13 +01:00
Ruben Hensen 855317b84d freshrss: pin the alpine/git init container 2026-10-25 04:32:13 +01:00
Ruben Hensen d69f2638d1 docs: record the vulnerability scan and the old Longhorn instance manager 2026-10-25 04:29:57 +01:00
Ruben Hensen 90e552c9f5 docs: update the roadmap's state 2026-10-25 04:25:43 +01:00
Ruben Hensen ad14995ba0 docs: record the wider Renovate coverage 2026-10-25 04:25:10 +01:00
Ruben Hensen c359384ade renovate: look at every app folder, and at CNPG images 2026-10-25 04:24:56 +01:00
Ruben Hensen a4fc386ee8 docs: record the new probes 2026-10-25 04:22:28 +01:00
Ruben Hensen 0dc0bc4322 docs: authentik's base URL is already set 2026-10-25 04:22:06 +01:00
Ruben Hensen d843f31d86 serpbear: add health probes 2026-10-25 04:19:29 +01:00
Ruben Hensen da23357946 freshrss: add health probes 2026-10-25 04:19:29 +01:00
Ruben Hensen b9d88f41b9 gitea: add health probes 2026-10-25 04:19:29 +01:00
Ruben Hensen 3c4a8cc8db docs: bring the authentik README up to date 2026-10-25 04:16:48 +01:00
Ruben Hensen d17e78130d docs: record metrics-server and plan the restore test 2026-10-25 04:15:43 +01:00
Ruben Hensen 22c5a325d6 Add metrics-server 2026-10-25 04:11:59 +01:00
Ruben Hensen ef07307f93 renovate: keep CI's Helm at Argo CD's version 2026-10-25 04:10:30 +01:00
Ruben Hensen aa940134ea docs: record the CI and the Renovate triage 2026-10-25 04:10:14 +01:00
Ruben Hensen f86ed09cd8 renovate: drop the github-tags manager, which duplicated the argocd one 2026-10-25 04:09:52 +01:00
Ruben Hensen 6d2bc379ba ci: retry when a registry rate-limits 2026-10-25 04:06:53 +01:00
26 changed files with 824 additions and 51 deletions

No files matched your search

+8
View File
@@ -99,6 +99,10 @@ spec:
- name: REQUESTS_CA_BUNDLE
value: /etc/ssl/certs/ca-certificates.crt
server:
resources:
requests:
cpu: 70m
memory: 560Mi
volumes:
- name: patched-ca-bundle
configMap:
@@ -184,6 +188,10 @@ spec:
mountPath: /etc/ssl/certs
readOnly: true
worker:
resources:
requests:
cpu: 100m
memory: 336Mi
volumes:
- name: patched-ca-bundle
configMap:
+14
View File
@@ -34,6 +34,20 @@ spec:
- name: patched-ca-bundle
mountPath: /etc/ssl/certs
readOnly: true
resources:
requests:
cpu: 10m
memory: 48Mi
cainjector:
resources:
requests:
cpu: 10m
memory: 80Mi
webhook:
resources:
requests:
cpu: 10m
memory: 32Mi
syncPolicy:
syncOptions:
- CreateNamespace=true
+4
View File
@@ -24,6 +24,10 @@ spec:
additionalArgs:
- --leader-lease-duration=60
- --leader-renew-deadline=40
resources:
requests:
cpu: 10m
memory: 48Mi
destination:
server: https://kubernetes.default.svc
namespace: cnpg-system
+16
View File
@@ -18,6 +18,22 @@ spec:
- chart: external-secrets
repoURL: https://charts.external-secrets.io
targetRevision: 2.13.0
helm:
values: |
resources:
requests:
cpu: 20m
memory: 64Mi
certController:
resources:
requests:
cpu: 10m
memory: 64Mi
webhook:
resources:
requests:
cpu: 10m
memory: 32Mi
destination:
server: https://kubernetes.default.svc
namespace: external-secrets
+34
View File
@@ -0,0 +1,34 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: metrics-server
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
chart: metrics-server
repoURL: https://kubernetes-sigs.github.io/metrics-server/
targetRevision: 3.14.0
helm:
values: |
# Resource usage for kubectl top and for sizing requests and limits.
# It keeps everything in memory, so it adds no writes to the SSD.
args:
# microk8s's kubelet serves a self-signed certificate; its own
# metrics-server addon passes the same flag.
- --kubelet-insecure-tls
resources:
requests:
cpu: 20m
memory: 64Mi
limits:
memory: 256Mi
destination:
server: https://kubernetes.default.svc
namespace: kube-system
syncPolicy:
automated:
prune: true
selfHeal: true
+1 -1
View File
@@ -7,7 +7,7 @@ spec:
project: default
source:
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
targetRevision: v8.7.0
targetRevision: v8.2.1
path: deploy/kubernetes/snapshot-controller
kustomize:
patches:
+2 -1
View File
@@ -21,4 +21,5 @@ patches:
- path: overlays/argocd-cm.patch.yaml
- path: overlays/argocd-rbac-cm.patch.yaml
- path: overlays/argocd-cmd-params-cm.patch.yaml
- path: overlays/patched-ca-bundle.patch.yaml
- path: overlays/patched-ca-bundle.patch.yaml
- path: overlays/resources.patch.yaml
+100
View File
@@ -0,0 +1,100 @@
# Requests from what the components used on 2026-10-25 (kubectl top every
# 5 minutes; highest value plus 10%). No limits until there are daytime
# numbers, see docs/roadmap.md.
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: argocd-application-controller
spec:
template:
spec:
containers:
- name: argocd-application-controller
resources:
requests:
cpu: 150m
memory: 704Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-repo-server
spec:
template:
spec:
containers:
- name: argocd-repo-server
resources:
requests:
cpu: 100m
memory: 128Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-server
spec:
template:
spec:
containers:
- name: argocd-server
resources:
requests:
cpu: 10m
memory: 64Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-redis
spec:
template:
spec:
containers:
- name: redis
resources:
requests:
cpu: 20m
memory: 96Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-applicationset-controller
spec:
template:
spec:
containers:
- name: argocd-applicationset-controller
resources:
requests:
cpu: 10m
memory: 48Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-notifications-controller
spec:
template:
spec:
containers:
- name: argocd-notifications-controller
resources:
requests:
cpu: 10m
memory: 48Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: argocd-dex-server
spec:
template:
spec:
containers:
- name: dex
resources:
requests:
cpu: 10m
memory: 32Mi
+26 -3
View File
@@ -14,7 +14,11 @@ Authentik is deployed as the central identity provider, providing OIDC and LDAP
- `blueprint-argocd-oidc.yaml` — OIDC provider for ArgoCD
- `blueprint-freshrss-oidc.yaml` — OIDC provider for FreshRSS
- `blueprint-actualbudget-proxy.yaml` — Proxy provider for Actual Budget
- `blueprint-longhorn-oidc.yaml` — OIDC provider for Longhorn (Envoy Gateway SecurityPolicy)
- `blueprint-scouting.yaml` — immich's login with Scouts Online, see docs/immich-scouts-online.md
- `blueprint-session-duration.yaml` — Session lifetime of the default authentication flow
- **NetworkPolicies** (`networkpolicy.yaml`) — the pods reach each other, DNS and the internet,
not the LAN or other namespaces; the worker and the database also reach the API server
## Session lifetime
@@ -30,9 +34,23 @@ Apache mod_auth_openidc refreshes on every request.
## LDAP Outpost
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by:
- **SOGo-mail** (in-cluster) — connects via `ak-outpost-ldap-outpost.authentik.svc.cluster.local:3389`
- **Stalwart** (NixOS, external) — connects via `ldap.rubenhensen.nl:389`
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by
**Stalwart** (NixOS on node001), which connects via `ldap.rubenhensen.nl:389`.
The outpost itself talks to the server through `https://authentik.rubenhensen.nl/`.
Authentik's worker deploys the outpost and keeps its version equal to the
server's. After the upgrades to 2026.5 and 2026.8 it didn't follow on its
own; this rolls it over:
```bash
kubectl -n authentik exec deploy/authentik-worker -- ak shell -c '
from authentik.outposts.models import Outpost
from authentik.outposts.tasks import outpost_controller
outpost_controller.send(Outpost.objects.get(name="LDAP outpost").pk)'
```
Upgrades go one release at a time (2026.2 → 2026.5 → 2026.8); since 2026.8
authentik refuses to skip one.
### NodePort Service
@@ -87,3 +105,8 @@ microk8s config > ~/.kube/config
```
**Note:** This must be re-applied after MicroK8s upgrades or cert rotations, as MicroK8s may regenerate the CA without the extension. The fix needs to run on every node in the cluster.
Checked 2026-10-25: the CA from this fix (`CN=microk8s-ca`, 2026-03-21, valid
until 2036) still has the extension after the refreshes to 1.34, 1.35 and
1.36. Check with
`openssl x509 -in /var/snap/microk8s/current/certs/ca.crt -noout -ext keyUsage`.
+4
View File
@@ -8,6 +8,10 @@ spec:
# start an in-place major upgrade. CNPG filled this in at creation.
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
primaryUpdateStrategy: unsupervised
resources:
requests:
cpu: 50m
memory: 256Mi
bootstrap:
initdb:
database: authentik
+14
View File
@@ -1,5 +1,19 @@
# Velero Backup Testing Guide
**Use `restore-test.sh`.** It restores a few namespaces from a real backup
into a throwaway k3s in Docker, from a read-only copy of the bucket, and
checks that every volume comes back with the bytes it was backed up with:
```bash
backup-testing/restore-test.sh # newest daily backup; ntfy, freshrss, actual-budget
backup-testing/restore-test.sh <backup> <namespace>... # something else
```
How it works and why it copies instead of pointing Velero at the NAS is at
the top of the script. The rest of this README describes the older k3d
scripts, which predate Envoy Gateway and still expect `*.hensen.io` and
sol-auth.
This guide explains how to test your Velero backups using a temporary K3d cluster.
## Features
+212
View File
@@ -0,0 +1,212 @@
#!/bin/bash
# Restore a few namespaces from the real Velero backups into a throwaway
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
# changes:
# - one backup (backups/<name>/) and the kopia repositories of the chosen
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with
# rclone into a Docker volume. A second Velero on the real bucket could
# run kopia maintenance on the same repositories as the cluster's own
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
# ReadOnly backup storage location
# - storage class longhorn becomes local-path; only core workload
# resources are restored (no routes, certificates, ExternalSecrets or
# NetworkPolicies)
# - every pod volume restore has to complete with as many bytes as its
# backup had, and the restored deployments have to become available
#
# Usage (on a machine with docker, kubectl and jq):
# backup-testing/restore-test.sh [backup] [namespace ...]
# Defaults: the newest Completed daily-backup, and ntfy freshrss
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
#
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
# into a temporary directory, and the copied backup holds every Secret of
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
# cluster (kubeconfig in the printed directory) until you run it with
# CLEANUP=1. FROM_VOLUME=<docker volume> takes the bucket from a volume
# instead of the NAS (for rehearsals; laid out as <bucket>/backups/...).
#
# Rehearsed on 2026-10-25 against a backup made in the test cluster itself,
# which is how two pitfalls showed up: Velero skips hostPath volumes, and
# k3s only provisions local PVs (which it can back up and restore) for a
# storage class with the annotation defaultVolumeType: local.
set -euo pipefail
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
MINIO=${MINIO:-http://192.168.1.99:9000}
BUCKET=velero-backups
PORT=${PORT:-16443}
K3S=rancher/k3s:v1.34.9-k3s1
RCLONE=rclone/rclone:1.75.1
HELM=alpine/helm:3.19.0
VELERO_CHART=12.2.1
AWS_PLUGIN=velero/velero-plugin-for-aws:v1.14.4
NET=restore-test VOL=restore-test-s3
cleanup() {
docker rm -f -v restore-k3s restore-s3 >/dev/null 2>&1 || true
docker volume rm "$VOL" >/dev/null 2>&1 || true
docker network rm "$NET" >/dev/null 2>&1 || true
[[ -n ${WORK:-} ]] && rm -rf "$WORK"
}
if [[ ${CLEANUP:-} == 1 ]]; then
cleanup; rm -rf "${TMPDIR:-/tmp}"/restore-test.*; echo "cleaned up"; exit 0
fi
if docker ps -a --format '{{.Names}}' | grep -qx restore-k3s; then
echo "A test cluster from an earlier run exists; run with CLEANUP=1 first." >&2
exit 1
fi
WORK=$(mktemp -d "${TMPDIR:-/tmp}/restore-test.XXXXXX")
chmod 700 "$WORK"
[[ ${KEEP:-} == 1 ]] || trap cleanup EXIT
pk() { kubectl --kubeconfig "$PROD_KUBECONFIG" "$@"; }
tk() { kubectl --kubeconfig "$WORK/kubeconfig" "$@"; }
# wait_for <seconds> <what> <command...>
wait_for() {
local limit=$1 what=$2; shift 2
local end=$((SECONDS + limit))
until "$@" >/dev/null 2>&1; do
if ((SECONDS > end)); then echo "timed out waiting for $what" >&2; exit 1; fi
sleep 5
done
}
backup=${1:-}
shift || true
namespaces=("$@")
[[ ${#namespaces[@]} -gt 0 ]] || namespaces=(ntfy freshrss actual-budget)
if [[ -z $backup ]]; then
backup=$(pk -n velero get backups.velero.io -o json | jq -r '[.items[]
| select(.metadata.labels["velero.io/schedule-name"] == "daily-backup" and .status.phase == "Completed")]
| max_by(.status.startTimestamp).metadata.name')
fi
echo "== backup $backup, namespaces: ${namespaces[*]}"
docker network create "$NET" >/dev/null
docker volume create "$VOL" >/dev/null
if [[ -n ${FROM_VOLUME:-} ]]; then
echo "== copy from the volume $FROM_VOLUME"
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
else
echo "== copy from the NAS (read-only)"
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env")
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
copy "backups/$backup"
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
rm -f "$WORK/nas.env"
fi
echo "== local S3 and k3s"
docker run -d --name restore-s3 --network "$NET" -v "$VOL:/data" "$RCLONE" \
serve s3 /data --addr :8080 --auth-key test,restore-test-secret >/dev/null
s3ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" restore-s3)
docker run -d --name restore-k3s --privileged --network "$NET" -p "127.0.0.1:$PORT:6443" "$K3S" \
server --disable traefik --disable metrics-server --tls-san 127.0.0.1 --tls-san restore-k3s >/dev/null
node_ready() { docker exec restore-k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; }
wait_for 180 "k3s" node_ready
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$PORT#" > "$WORK/kubeconfig"
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed 's#https://127.0.0.1:6443#https://restore-k3s:6443#' > "$WORK/kubeconfig-net"
echo "== Velero"
cat > "$WORK/values.yaml" <<EOF
configuration:
backupStorageLocation:
- name: default
provider: aws
bucket: $BUCKET
accessMode: ReadOnly
config:
region: minio
s3ForcePathStyle: "true"
s3Url: http://$s3ip:8080
volumeSnapshotLocation: []
defaultRepoMaintainFrequency: 8760h
credentials:
secretContents:
cloud: |
[default]
aws_access_key_id=test
aws_secret_access_key=restore-test-secret
initContainers:
- name: velero-plugin-for-aws
image: $AWS_PLUGIN
volumeMounts:
- mountPath: /target
name: plugins
deployNodeAgent: true
snapshotsEnabled: false
upgradeCRDs: false
EOF
docker run --rm --network "$NET" -e HOME=/tmp -e KC="$(base64 < "$WORK/kubeconfig-net")" \
-e VALUES="$(base64 < "$WORK/values.yaml")" --entrypoint sh "$HELM" -c '
echo "$KC" | base64 -d > /tmp/kc && echo "$VALUES" | base64 -d > /tmp/v.yaml &&
helm --kubeconfig /tmp/kc install velero velero --repo https://vmware-tanzu.github.io/helm-charts \
--version '"$VELERO_CHART"' -n velero --create-namespace -f /tmp/v.yaml >/dev/null'
tk apply -f - >/dev/null <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: change-storage-class
namespace: velero
labels:
velero.io/plugin-config: ""
velero.io/change-storage-class: RestoreItemAction
data:
longhorn: local-path
EOF
wait_for 300 "Velero to sync $backup" tk -n velero get backups.velero.io "$backup"
echo " Velero sees $backup"
echo "== restore"
tk apply -f - >/dev/null <<EOF
apiVersion: velero.io/v1
kind: Restore
metadata:
name: restore-test
namespace: velero
spec:
backupName: $backup
includedNamespaces: [$(IFS=,; echo "${namespaces[*]}")]
# PersistentVolumes have to be included: only then does Velero clear the
# PVC's volumeName. It doesn't restore a PV whose data comes from kopia;
# k3s provisions a new one.
includedResources: [namespaces, persistentvolumeclaims, persistentvolumes, pods, deployments,
replicasets, statefulsets, services, configmaps, secrets, serviceaccounts]
restorePVs: true
EOF
restore_done() { [[ $(tk -n velero get restore restore-test -o jsonpath='{.status.phase}') =~ ^(Completed|PartiallyFailed|Failed|FailedValidation)$ ]]; }
wait_for 1800 "the restore" restore_done
phase=$(tk -n velero get restore restore-test -o jsonpath='{.status.phase}')
echo " restore: $phase"
echo "== check"
failed=0
expected=$(docker run --rm -v "$VOL:/data" alpine:3.22 zcat "/data/$BUCKET/backups/$backup/$backup-podvolumebackups.json.gz" |
jq -r --arg ns "$(IFS=,; echo "${namespaces[*]}")" '.[] | select(.spec.pod.namespace as $p | $ns | split(",") | index($p))
| "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0)"' | sort -u)
restored=$(tk -n velero get podvolumerestores.velero.io -o json |
jq -r '.items[] | "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0) \(.status.phase)"' | sort -u)
while read -r vol bytes; do
[[ -z $vol ]] && continue
got=$(awk -v v="$vol" '$1 == v {print $2, $3}' <<<"$restored")
if [[ $got == "$bytes Completed" ]]; then
echo " ok $vol ($bytes bytes)"
else
echo " FAILED $vol: backed up $bytes bytes, restore: ${got:-none}"; failed=1
fi
done <<<"$expected"
for ns in "${namespaces[@]}"; do
if tk -n "$ns" wait --for=condition=Available deployment --all --timeout=300s >/dev/null 2>&1; then
echo " ok $ns: deployments available"
else
echo " FAILED $ns: deployments not available"; tk -n "$ns" get pods; failed=1
fi
done
[[ $phase == Completed ]] || failed=1
[[ ${KEEP:-} == 1 ]] && echo "Kept; kubeconfig: $WORK/kubeconfig. Remove with CLEANUP=1 $0"
if [[ $failed == 0 ]]; then echo "== restore test passed"; else echo "== restore test FAILED"; exit 1; fi
+10
View File
@@ -54,6 +54,16 @@ between them live.
on immich-claim is one 233 GiB snapshot (`system-backup-0b7f8c6047294b36`,
from the Longhorn system backup of 2026-10-22) that everything merged
into; 148 snapshots hold 273 GiB across all volumes.
- [ ] Delete the snapshots the Longhorn system backups left behind: 116
`system-backup-*` snapshots (one per volume for each system backup
before an upgrade step on 2026-10-22 and 23) hold 258 GiB. The system
backups on the NAS don't need them. The small volumes are cheap; on
immich-claim the 233 GiB one is the base the rest merged into, so
deleting it means hours of merging like on 2026-10-24: one night, or
after the move to the NVMe drive. Then trim (next item); deleted files
only free space once both are done. Next time, take system backups with
the volume backup policy `disabled` when the nightly backups are fresh,
so they create no snapshots
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
so blocks of deleted files are released.
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
+158 -20
View File
@@ -8,11 +8,14 @@ bottom.
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
35 Argo CD apps, all Synced/Healthy; every Longhorn volume has one
replica. Backups go to the NAS (192.168.1.99): Longhorn to NFS at 00:00
(7 kept), Velero to MinIO at 01:00 (daily kept 30 days, weekly 90 days).
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
through ntfy. node001 runs mail and tunnels the public hosts home.
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
and s2 (home) watch each other and alert through ntfy; s2 also gets an
hourly report on the backups and Vault's seal from inside the cluster.
Every app namespace has NetworkPolicies, and CI renders and validates all
apps on every PR. node001 runs mail and tunnels the public hosts home.
## Follow-ups from this week
@@ -77,7 +80,25 @@ friend.
the NAS
- [ ] Decide which copy of the immich library goes off-site (see 5)
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
Planned on 2026-10-25, not run: a second Velero on the real bucket is
risky even with a ReadOnly location, because kopia could then run
maintenance on the same repositories from two clusters. Instead: copy
one backup (`backups/<name>/`) and the kopia repositories of a few small
namespaces (`kopia/{gatus,ntfy,freshrss}/`) read-only from the NAS with
rclone into a local folder, serve that with `rclone serve s3`, and
restore into a throwaway k3s in Docker (Velero 1.18.2, node agent on,
storage class longhorn mapped to local-path, without httproutes,
certificates, referencegrants and networkpolicies). It needs the MinIO
key outside the cluster, which wants your go-ahead; better still, a
MinIO user with read-only access to `velero-backups`.
Built as `backup-testing/restore-test.sh` (2026-10-25) and rehearsed
end to end against a backup made inside the test cluster, with a
ReadOnly location: data restored byte for byte (same sha256), deployment
available, everything removed afterwards. Two pitfalls found on the way
are in the script's header. Not yet run against the real backups:
- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from
the cluster), then schedule it
### 3. authentik 2026.2.1 → 2026.8.5
@@ -105,10 +126,13 @@ April and May.
- [x] Log in once through each app: immich through Scouts Online, Argo CD,
Vault, Longhorn, FreshRSS, Actual, mail
- [ ] After a week without problems: delete the dump and the snapshot
- [ ] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`) becomes
required in 2026.11. authentik answers on authentik.rubenhensen.nl and
auth.scoutingmbg.nl (immich's issuer), so find out first what the
setting does with two hostnames
- [x] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`), required
from 2026.11: already `https://authentik.rubenhensen.nl`, filled in by
the 2026.8 migration from the embedded outpost's host. In 2026.8.5 it is
only stored; on main it fills itself in the same way and sets the
embedded outpost's host from it. OIDC issuers still come from the
request's host, so immich's issuer on auth.scoutingmbg.nl is unaffected
(checked 2026-10-25 in the source)
### 4. Vault 1.17.2 → 2.2.1
@@ -179,15 +203,21 @@ until 2026-10-21 without anyone noticing.
- [x] A stopgap for the worst blind spots (2026-10-25,
`gatus/backup-check.yaml`): an hourly CronJob reads the API and pushes
four external endpoints to s2 (group "cluster"): the newest daily and
six external endpoints to s2 (group "cluster"): the newest daily and
weekly Velero backup Completed and recent, every attached Longhorn
volume healthy with a backup younger than 26h, and no Vault pod sealed.
volume healthy with a backup younger than 26h, no Vault pod sealed,
every Argo CD app Synced and Healthy, every CNPG cluster healthy.
Gatus alerts through ntfy after two failed reports, or when no report
arrives for 3 hours. The token comes from an ESO Password generator.
Tested with made-up data for each failure (a Failed or old backup, a
degraded or unbacked-up volume, a sealed pod)
- [x] metrics-server (2026-10-25, `apps/templates/metrics-server-helm.yaml`):
`kubectl top` works; it keeps its data in memory, so it adds no disk
writes. The first look: immich v3 checks file checksums every night from
03:00 UTC for up to an hour (IntegrityService), reading ~100 MB/s from
sda; it doesn't overlap the backups at 00:00 and 01:00
- [ ] After the second SSD (a time series database writes all the time
and would add to today's stalls): metrics-server, then Prometheus or
and would add to today's stalls): Prometheus or
VictoriaMetrics with Grafana, alerting to ntfy. Loki later
- [ ] Alerts worth having: disk latency and fill, lease losses and
controller restarts, backups that don't complete, Longhorn volumes that
@@ -237,7 +267,12 @@ Counted 2026-10-25 over the 73 running containers:
without a policy
- [ ] Requests and memory limits, the big ones first (immich, authentik,
Argo CD's repo-server)
- [ ] Probes
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
SSD's stalls. Still without probes: authentik's LDAP outpost (deployed
by authentik), the ESO controller, dex and Argo CD's redis (upstream
manifests)
- [ ] Non-root where the image allows it
### 8. Renovate and CI
@@ -245,9 +280,68 @@ Counted 2026-10-25 over the 73 running containers:
10 Renovate PRs are open, the oldest from February, and nothing checks a
PR before Argo CD applies it.
- [ ] A GitHub Actions workflow that renders the Helm apps and runs
kubeconform on every PR, with conftest for this repo's rules
- [ ] Automerge patch versions of low-risk apps once that runs
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
does (the apps chart, every Application's chart with its values, value
files and parameters, kustomizations, plain folders, and
external-snapshotter cloned with its patches) and validates the result
with kubeconform (strict, CRD schemas from the datree catalog), plus a
duplicate-key check on every YAML file. Helm 4.2.1 like Argo CD 3.5,
versions pinned and checksummed, kept current by Renovate through the
`# renovate:` comments. Tested by planting a typo in a Deployment, a
chart version that doesn't exist and a value the immich chart's schema
rejects; all three fail the run. Locally:
`docker run --rm -v $PWD:/repo:ro <image with helm, kustomize, kubeconform, PyYAML> python /repo/tools/ci/validate.py`
- It found `syncPolicy.automated` twice in
`apps/templates/snapshot-controller-helm.yaml` (once with `prune: true`,
added 2025-03-03); Argo CD kept the last one, without prune. Now listed
once, as it ran. Prune was probably meant; turn it on deliberately if so
- [x] This repo's own rules, in `tools/ci/validate.py` rather than
conftest (2026-10-25), on what the repo's own folders render: no Secret
with data in git, and every image has a tag other than latest (both
fail the run; tested with a planted Secret and an untagged image), plus
PR annotations for containers without requests or probes. freshrss's
`alpine/git` init container had no tag; now `v2.52.0`
- [ ] Automerge patch versions of low-risk apps now that CI runs
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
`argocd` manager and a custom `github-tags` regex both matched. The
regex is gone; Renovate should close #15 by itself
- [x] Renovate only looked at six folders for images and only at
apps/templates for Applications, so it never offered updates for the
vault, immich, velero, mariadb-operator and actualbudget charts, or for
images in freshrss, mbgwp, dns, vault and immich, or for CNPG's
`imageName` (authentik still runs PostgreSQL 17.4). Both managers now
cover every app folder, a regex manager reads `imageName` in
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
(those need pg_upgrade or a dump). Expect a batch of new PRs
(2026-10-25; `renovate-config-validator` passes)
- [ ] The open PRs, all validated on 2026-10-25 against main as it is now
(rebases requested, so the checks show on the PRs once Renovate has
pushed). Merging is yours; merging deploys:
- #28 cert-manager v1.21.3: security release, "All users should
upgrade" (drops the compromised eMudhra root, fixes an ARI renewal
loop and a private key copy path). Merge first
- #24 external-snapshotter v8.2.1 → v8.7.0: 8.6 moved VolumeGroupSnapshot
to v1 and 8.7 turns its feature gate on; the cluster already serves the
v1 CRDs. Merge; close #15
- #25 WordPress chart 0.15.0 → 0.16.7: the image is pinned
(7.1.3-php8.3-apache), so the render only changes labels and
checksums (a pod restart). Merge; #19 (0.15.1) is superseded
- #26 Gitea 1.27.3 → 28.2.0 (Gitea renumbered after 1.27): security
fixes; the breaking changes are to egress host lists
(`ALLOWED_DOMAINS` and friends), none of which are set here. Database
migrations can't be undone, so merge after a fresh backup
- #23 ntfy v2.28.0 → v2.34.0: six releases in 17 days; 2.31.0 had a
database load bug that ntfy.sh rolled back (fixed in 2.31.1). No
breaking migrations. Fine, or wait a few days for 2.34 to settle
- #17 external-secrets 2.13.0 → 2.14.0: no breaking changes in the
notes; every secret goes through it, so merge when there's time to
watch a force-sync
- #27 go-dqlite v3.0.4 for `tools/dqlite-vacuum`: builds; harmless
- #29 yaml.v3 for the same tool: builds, but the code imports yaml.v2,
so it only adds an unused requirement. Close
- #31 python 3.15.1 for `gatus/backup-check.yaml`: the check's tests
(made-up data for every failure) pass on that image. Merge
### 9. One node
@@ -285,9 +379,53 @@ step after that, if downtime starts to matter.
- [ ] opti1, with sudo: `/root/backend-*` and `/root/dqlite-pre-*` (from
before 1.33, 1.34 and the VACUUM; 1.34 migrated the schema, so none of
them fits 1.36) and `~/containerd.bak` (root-owned, from 2025-02)
- [ ] `~/cadir` on opti1 holds a CA key and certificate from 2026-03-21
that are not microk8s's current CA. Find out whether anything still
trusts it; if not, delete it
- [ ] `rm -r ~/cadir` on opti1: a CA key and certificate made four minutes
before the real keyUsage fix on 2026-03-21 (OpenSSL's default subject
"Internet Widgits Pty Ltd", valid for a year) and never installed;
microk8s's CA is the `CN=microk8s-ca` one from 19:16 that day, valid
until 2036
### 11. Known vulnerabilities
A one-off scan on 2026-10-25 of all 52 images running in the cluster
(Trivy 0.75.0, remote, linux/amd64, HIGH and CRITICAL only). Counts are
critical / high:
| Image | Crit | High | What to do |
|---|---|---|---|
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; goes with the cleanup from 2026-10-27 |
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. Renovate now offers 17.7; a `17.x-standard-trixie` image is more current |
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
| coredns/coredns:1.10.1 | 5 | 70 | see below |
| dexidp/dex:v2.45.1, argocd v3.5.6, redis 8.2.3 | 4, 1, 2 | | come with Argo CD |
| mariadb:11.4.5 | 4 | 65 | the newest 11.4 patch; Renovate now covers it |
| calico v3.29.3 | 2-3 | ~50 | comes with microk8s |
| goauthentik ldap / server 2026.8.5 | 3, 1 | 74, 65 | the next authentik patch |
Nothing critical in Envoy, Vault, Velero, cert-manager, ESO, ntfy, gatus or
metrics-server.
- [ ] 16 volumes still have their engine in Longhorn 1.7.3's instance
manager (`instance-manager-07d66a5…`, running since March): vault ×3,
authentik-postgres, immich-postgres (PG14), mbgwp (mariadb and
wordpress), gitea, freshrss ×3, ntfy, gatus, serpbear, bolletjestrui,
actual-budget. On one node a pod restart never detaches the volume (the
new pod lands on the same node), so only a real detach moves the engine
to 1.13's instance manager: scale to 0, wait until Longhorn shows the
volume detached, scale back (Argo CD's selfHeal undoes a scale-down, so
pause it for the app). Vault comes back sealed; CNPG clusters want
`cnpg.io/hibernation=on` instead of a scale-down. The old instance
manager goes away by itself once it serves nothing
- [ ] CoreDNS runs 1.10.1 from when the addon was enabled; the addon
repository has 1.14.3. `sudo microk8s disable dns && sudo microk8s enable dns:1.1.1.1`
(a short DNS outage for the whole cluster; check the Corefile after)
- [ ] The Trivy Operator for scans that keep running (on the wishlist)
## wishlist.txt (2026-01-25)
+18 -1
View File
@@ -23,7 +23,7 @@ spec:
io.kompose.service: freshrss
spec:
initContainers:
- image: alpine/git
- image: alpine/git:v2.52.0
name: git
command:
- git
@@ -79,6 +79,23 @@ spec:
ports:
- containerPort: 80
protocol: TCP
resources:
requests:
cpu: 50m
memory: 176Mi
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
# browser.
readinessProbe:
tcpSocket:
port: 80
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 80
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
volumeMounts:
- mountPath: /var/www/FreshRSS/data
name: data
+28
View File
@@ -11,6 +11,8 @@
# younger than 26h (detached volumes are skipped; the recurring job skips
# them too)
# - vault-unsealed: all Vault pods report vault-sealed=false
# - argocd-apps: every Argo CD application is Synced and Healthy
# - postgres: every CloudNativePG cluster is in a healthy state
#
# The token is generated by ESO and never leaves the cluster; Gatus reads
# it from the same Secret.
@@ -63,6 +65,12 @@ rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["list"]
- apiGroups: ["argoproj.io"]
resources: ["applications"]
verbs: ["list"]
- apiGroups: ["postgresql.cnpg.io"]
resources: ["clusters"]
verbs: ["list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
@@ -170,6 +178,24 @@ data:
return f"sealed: {', '.join(sealed)}" if sealed else None
def argocd():
bad = []
for a in items("/apis/argoproj.io/v1alpha1/namespaces/argocd/applications"):
s = a.get("status", {})
sync = s.get("sync", {}).get("status")
health = s.get("health", {}).get("status")
if sync != "Synced" or health != "Healthy":
bad.append(f"{a['metadata']['name']} {sync}/{health}")
return "; ".join(bad) or None
def postgres():
bad = [f"{c['metadata']['namespace']}/{c['metadata']['name']}: {c.get('status', {}).get('phase')}"
for c in items("/apis/postgresql.cnpg.io/v1/clusters")
if c.get("status", {}).get("phase") != "Cluster in healthy state"]
return "; ".join(bad) or None
def push(key, error):
query = {"success": "false" if error else "true"}
if error:
@@ -185,6 +211,8 @@ data:
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
"cluster_longhorn": lambda: longhorn(26),
"cluster_vault-unsealed": vault,
"cluster_argocd-apps": argocd,
"cluster_postgres": postgres,
}
failed_pushes = 0
for key, check in checks.items():
+12
View File
@@ -169,3 +169,15 @@ external-endpoints:
heartbeat:
interval: 3h
alerts: *cluster-alert
- name: argocd-apps
group: cluster
token: ${BACKUP_CHECK_TOKEN}
heartbeat:
interval: 3h
alerts: *cluster-alert
- name: postgres
group: cluster
token: ${BACKUP_CHECK_TOKEN}
heartbeat:
interval: 3h
alerts: *cluster-alert
+19
View File
@@ -50,6 +50,25 @@ spec:
- containerPort: 3000
name: http
protocol: TCP
resources:
requests:
cpu: 20m
memory: 336Mi
# /api/healthz checks the database and the cache.
readinessProbe:
httpGet:
path: /api/healthz
port: http
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /api/healthz
port: http
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
volumeMounts:
- mountPath: /data
name: data
+14
View File
@@ -63,6 +63,10 @@ spec:
main:
containers:
main:
resources:
requests:
cpu: 10m
memory: 64Mi
image:
repository: docker.io/valkey/valkey
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
@@ -85,6 +89,10 @@ spec:
image:
repository: ghcr.io/immich-app/immich-server
pullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 1280Mi
# Node trusts its built-in roots, which still include the
# ones above the compromised eMudhra intermediates. Use
# patched-ca-bundle through OpenSSL until Node ships a fix.
@@ -109,6 +117,12 @@ spec:
image:
repository: ghcr.io/immich-app/immich-machine-learning
pullPolicy: IfNotPresent
# Idle at night (234Mi); the models take over a GiB when
# it works.
resources:
requests:
cpu: 50m
memory: 1Gi
env:
TRANSFORMERS_CACHE: /cache
HF_XET_CACHE: /cache/huggingface-xet
+4
View File
@@ -9,6 +9,10 @@ spec:
# CNPG image with VectorChord and pgvector, which immich 2.x and 3.x use
# for its vector indexes (pgvecto.rs is gone in immich 3.0).
imageName: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
resources:
requests:
cpu: 20m
memory: 320Mi
postgresql:
shared_preload_libraries:
+12
View File
@@ -16,14 +16,26 @@ spec:
targetRevision: 25.10.4
helm:
values: |
resources:
requests:
cpu: 10m
memory: 48Mi
metrics:
enabled: true
serviceMonitor:
enabled: false
webhook:
resources:
requests:
cpu: 10m
memory: 32Mi
serviceMonitor:
enabled: false
certController:
resources:
requests:
cpu: 10m
memory: 32Mi
serviceMonitor:
enabled: false
syncPolicy:
+32 -22
View File
@@ -6,7 +6,7 @@
],
"argocd": {
"managerFilePatterns": [
"/apps/templates/.+\\.yaml$/"
"/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
]
},
"helm-values": {
@@ -16,25 +16,10 @@
},
"kubernetes": {
"managerFilePatterns": [
"/apps/templates/.+\\.yaml$/",
"/serpbear/.+\\.yaml$/",
"/bolletjestrui/.+\\.yaml$/",
"/gitea/.+\\.yaml$/",
"/gatus/.+\\.yaml$/",
"/ntfy/.+\\.yaml$/"
"/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
]
},
"customManagers": [
{
"customType": "regex",
"managerFilePatterns": [
"/apps/templates/.+\\.yaml$/"
],
"matchStrings": [
"repoURL: https://github.com/(?<depName>[^/]+/[^/\\.]+)\\.git\\n\\s+targetRevision: (?<currentValue>.*)"
],
"datasourceTemplate": "github-tags"
},
{
"customType": "regex",
"managerFilePatterns": [
@@ -44,6 +29,17 @@
"repoURL: (?<registryUrl>.*?)\\n\\s+targetRevision: (?<currentValue>[^\\s]+)\\n\\s+chart: (?<depName>[^\\s]+)"
],
"datasourceTemplate": "helm"
},
{
"customType": "regex",
"description": "CloudNativePG clusters name their image in imageName",
"managerFilePatterns": [
"/^[^/]+/postgresql-cluster[^/]*\\.yaml$/"
],
"matchStrings": [
"imageName: (?<depName>[^:\\s]+):(?<currentValue>\\S+)"
],
"datasourceTemplate": "docker"
}
],
"packageRules": [
@@ -56,12 +52,26 @@
"separateMinorPatch": true
},
{
"matchDatasources": [
"github-tags"
"description": "CI renders with the same Helm as Argo CD (3.5 ships Helm 4.2); move it together with Argo CD",
"matchFileNames": [
".github/workflows/**"
],
"semanticCommits": "enabled",
"commitMessageTopic": "{{depName}}",
"separateMinorPatch": true
"matchDepNames": [
"helm/helm"
],
"enabled": false
},
{
"description": "A PostgreSQL major upgrade needs pg_upgrade or a dump, not a tag bump",
"matchDepNames": [
"ghcr.io/cloudnative-pg/postgresql",
"ghcr.io/tensorchord/cloudnative-vectorchord",
"ghcr.io/tensorchord/cloudnative-pgvecto.rs"
],
"matchUpdateTypes": [
"major"
],
"enabled": false
}
]
}
+17
View File
@@ -56,6 +56,23 @@ spec:
ports:
- containerPort: 3000
protocol: TCP
resources:
requests:
cpu: 10m
memory: 160Mi
readinessProbe:
httpGet:
path: /login
port: 3000
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
tcpSocket:
port: 3000
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
volumeMounts:
- mountPath: /app/data
name: data
+57 -3
View File
@@ -15,6 +15,13 @@ in the app folders. Sources:
same, with the Application's kustomize patches applied
Every YAML file in the repo is also checked for duplicate keys, which
YAML parsers resolve silently by keeping the last one.
Repo rules, on what this repo's own folders render (not on charts or
remote bases):
- no Secret with data: secrets come from Vault through ESO
- every image has a tag other than latest
- warnings (annotations on the PR) for containers without requests or
probes
"""
import fnmatch
import os
@@ -22,6 +29,7 @@ import pathlib
import subprocess
import sys
import tempfile
import time
import yaml
@@ -38,7 +46,15 @@ failures = []
def run(cmd, **kw):
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
# Registries rate-limit now and then (ghcr.io answers 429); try again.
for attempt in range(3):
try:
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
except subprocess.CalledProcessError as e:
transient = any(s in (e.stderr or "") for s in ("429", "toomanyrequests", "timeout", "connection reset"))
if not transient or attempt == 2:
raise
time.sleep(10 * (attempt + 1))
class UniqueKeyLoader(yaml.SafeLoader):
@@ -78,7 +94,41 @@ def check_duplicate_keys():
failures.append(f"{path.relative_to(ROOT)}: {e}")
def validate(label, manifests):
def pod_specs(doc):
spec = doc.get("spec") or {}
kind = doc.get("kind")
if kind == "Pod":
yield spec
elif kind == "CronJob":
yield spec.get("jobTemplate", {}).get("spec", {}).get("template", {}).get("spec", {})
elif kind in ("Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"):
yield spec.get("template", {}).get("spec", {})
def check_rules(label, docs):
for d in docs:
name = f"{d.get('kind')}/{d.get('metadata', {}).get('name')}"
if d.get("kind") == "Secret" and (d.get("data") or d.get("stringData")):
failures.append(f"{label}: {name} has data in git; put it in Vault and use an ExternalSecret")
for spec in pod_specs(d):
containers = spec.get("containers", []) + spec.get("initContainers", [])
for c in containers:
image = c.get("image", "")
tag = image.split("@")[0].rsplit("/", 1)[-1].partition(":")[2]
if "@" not in image and tag in ("", "latest"):
failures.append(f"{label}: {name} container {c.get('name')} has no fixed tag ({image})")
for c in spec.get("containers", []):
missing = []
if not c.get("resources", {}).get("requests"):
missing.append("requests")
if d.get("kind") in ("Deployment", "StatefulSet", "DaemonSet") and not (
c.get("readinessProbe") or c.get("livenessProbe")):
missing.append("probes")
if missing:
print(f"::warning title={label}::{name} container {c.get('name')} has no {' or '.join(missing)}")
def validate(label, manifests, rules=False):
"""kubeconform on rendered manifests; returns the parsed documents."""
try:
docs = [d for d in yaml.safe_load_all(manifests) if d]
@@ -103,6 +153,8 @@ def validate(label, manifests):
for l in lines[:-1]:
print(f" {l}")
failures.append(f"{label}: {summary or result.stderr.strip()}")
if rules and docs:
check_rules(label, [d for d in docs if isinstance(d, dict)])
return docs or []
@@ -238,7 +290,9 @@ def main():
failures.append(f"{label}: render failed")
continue
if manifests is not None:
validate(label, manifests)
own = src.get("repoURL") in OWN_REPOS and not src.get("chart") and not (
ROOT / src.get("path", ".") / "Chart.yaml").exists()
validate(label, manifests, rules=own)
if failures:
print(f"\n{len(failures)} problem(s):")
+4
View File
@@ -3,6 +3,10 @@ global:
tlsDisable: true
injector:
enabled: true
resources:
requests:
cpu: 10m
memory: 32Mi
# The chart's default anti-affinity forbids two injector pods on one
# node, so on this single node a rolling update can never start the new
# pod while the old one runs. Recreate removes the old pod first, which
+4
View File
@@ -68,6 +68,10 @@ spec:
nodeAgent:
containerSecurityContext:
privileged: true
resources:
requests:
cpu: 10m
memory: 64Mi
# 512Mi got the server OOMKilled 314 times, including halfway
# through a backup of ~14.7k items.
resources: