mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 22:29:09 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
16a73e04bf |
No files matched your search
@@ -99,10 +99,6 @@ spec:
|
|||||||
- name: REQUESTS_CA_BUNDLE
|
- name: REQUESTS_CA_BUNDLE
|
||||||
value: /etc/ssl/certs/ca-certificates.crt
|
value: /etc/ssl/certs/ca-certificates.crt
|
||||||
server:
|
server:
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 70m
|
|
||||||
memory: 560Mi
|
|
||||||
volumes:
|
volumes:
|
||||||
- name: patched-ca-bundle
|
- name: patched-ca-bundle
|
||||||
configMap:
|
configMap:
|
||||||
@@ -188,10 +184,6 @@ spec:
|
|||||||
mountPath: /etc/ssl/certs
|
mountPath: /etc/ssl/certs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
worker:
|
worker:
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 336Mi
|
|
||||||
volumes:
|
volumes:
|
||||||
- name: patched-ca-bundle
|
- name: patched-ca-bundle
|
||||||
configMap:
|
configMap:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://charts.jetstack.io
|
repoURL: https://charts.jetstack.io
|
||||||
targetRevision: v1.21.3
|
targetRevision: v1.21.2
|
||||||
chart: cert-manager
|
chart: cert-manager
|
||||||
helm:
|
helm:
|
||||||
parameters:
|
parameters:
|
||||||
@@ -34,20 +34,6 @@ spec:
|
|||||||
- name: patched-ca-bundle
|
- name: patched-ca-bundle
|
||||||
mountPath: /etc/ssl/certs
|
mountPath: /etc/ssl/certs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 48Mi
|
|
||||||
cainjector:
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 80Mi
|
|
||||||
webhook:
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 32Mi
|
|
||||||
syncPolicy:
|
syncPolicy:
|
||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
|
|||||||
@@ -24,10 +24,6 @@ spec:
|
|||||||
additionalArgs:
|
additionalArgs:
|
||||||
- --leader-lease-duration=60
|
- --leader-lease-duration=60
|
||||||
- --leader-renew-deadline=40
|
- --leader-renew-deadline=40
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 48Mi
|
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: cnpg-system
|
namespace: cnpg-system
|
||||||
|
|||||||
@@ -17,23 +17,7 @@ spec:
|
|||||||
sources:
|
sources:
|
||||||
- chart: external-secrets
|
- chart: external-secrets
|
||||||
repoURL: https://charts.external-secrets.io
|
repoURL: https://charts.external-secrets.io
|
||||||
targetRevision: 2.14.0
|
targetRevision: 2.13.0
|
||||||
helm:
|
|
||||||
values: |
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 20m
|
|
||||||
memory: 64Mi
|
|
||||||
certController:
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 64Mi
|
|
||||||
webhook:
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 32Mi
|
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: external-secrets
|
namespace: external-secrets
|
||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://groundhog2k.github.io/helm-charts/
|
repoURL: https://groundhog2k.github.io/helm-charts/
|
||||||
targetRevision: 0.16.7
|
targetRevision: 0.15.1
|
||||||
chart: wordpress
|
chart: wordpress
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
@@ -101,23 +101,6 @@ spec:
|
|||||||
timeoutSeconds: 5
|
timeoutSeconds: 5
|
||||||
failureThreshold: 30
|
failureThreshold: 30
|
||||||
|
|
||||||
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
|
|
||||||
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
|
|
||||||
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
|
|
||||||
# crawler asking for 88 pages at once got the container OOMKilled on
|
|
||||||
# 2026-10-25. With 12 workers a burst waits in the listen backlog
|
|
||||||
# instead. Listen is the chart's default (containerPort).
|
|
||||||
apachePortsConfig: |
|
|
||||||
Listen 8000
|
|
||||||
<IfModule mpm_prefork_module>
|
|
||||||
StartServers 4
|
|
||||||
MinSpareServers 2
|
|
||||||
MaxSpareServers 6
|
|
||||||
ServerLimit 12
|
|
||||||
MaxRequestWorkers 12
|
|
||||||
MaxConnectionsPerChild 1000
|
|
||||||
</IfModule>
|
|
||||||
|
|
||||||
# Custom PHP configuration
|
# Custom PHP configuration
|
||||||
customPhpConfig: |
|
customPhpConfig: |
|
||||||
upload_max_filesize = 64M
|
upload_max_filesize = 64M
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
apiVersion: argoproj.io/v1alpha1
|
|
||||||
kind: Application
|
|
||||||
metadata:
|
|
||||||
name: metrics-server
|
|
||||||
namespace: argocd
|
|
||||||
finalizers:
|
|
||||||
- resources-finalizer.argocd.argoproj.io
|
|
||||||
spec:
|
|
||||||
project: default
|
|
||||||
source:
|
|
||||||
chart: metrics-server
|
|
||||||
repoURL: https://kubernetes-sigs.github.io/metrics-server/
|
|
||||||
targetRevision: 3.14.0
|
|
||||||
helm:
|
|
||||||
values: |
|
|
||||||
# Resource usage for kubectl top and for sizing requests and limits.
|
|
||||||
# It keeps everything in memory, so it adds no writes to the SSD.
|
|
||||||
args:
|
|
||||||
# microk8s's kubelet serves a self-signed certificate; its own
|
|
||||||
# metrics-server addon passes the same flag.
|
|
||||||
- --kubelet-insecure-tls
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 20m
|
|
||||||
memory: 64Mi
|
|
||||||
limits:
|
|
||||||
memory: 256Mi
|
|
||||||
destination:
|
|
||||||
server: https://kubernetes.default.svc
|
|
||||||
namespace: kube-system
|
|
||||||
syncPolicy:
|
|
||||||
automated:
|
|
||||||
prune: true
|
|
||||||
selfHeal: true
|
|
||||||
@@ -7,7 +7,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
||||||
targetRevision: v8.7.0
|
targetRevision: v8.2.1
|
||||||
path: deploy/kubernetes/snapshot-controller
|
path: deploy/kubernetes/snapshot-controller
|
||||||
kustomize:
|
kustomize:
|
||||||
patches:
|
patches:
|
||||||
|
|||||||
@@ -22,4 +22,3 @@ patches:
|
|||||||
- path: overlays/argocd-rbac-cm.patch.yaml
|
- path: overlays/argocd-rbac-cm.patch.yaml
|
||||||
- path: overlays/argocd-cmd-params-cm.patch.yaml
|
- path: overlays/argocd-cmd-params-cm.patch.yaml
|
||||||
- path: overlays/patched-ca-bundle.patch.yaml
|
- path: overlays/patched-ca-bundle.patch.yaml
|
||||||
- path: overlays/resources.patch.yaml
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
# Requests from what the components used on 2026-10-25 (kubectl top every
|
|
||||||
# 5 minutes; highest value plus 10%). No limits until there are daytime
|
|
||||||
# numbers, see docs/roadmap.md.
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: StatefulSet
|
|
||||||
metadata:
|
|
||||||
name: argocd-application-controller
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: argocd-application-controller
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 150m
|
|
||||||
memory: 704Mi
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: argocd-repo-server
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: argocd-repo-server
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 128Mi
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: argocd-server
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: argocd-server
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 64Mi
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: argocd-redis
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: redis
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 20m
|
|
||||||
memory: 96Mi
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: argocd-applicationset-controller
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: argocd-applicationset-controller
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 48Mi
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: argocd-notifications-controller
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: argocd-notifications-controller
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 48Mi
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: argocd-dex-server
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: dex
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 32Mi
|
|
||||||
+3
-26
@@ -14,11 +14,7 @@ Authentik is deployed as the central identity provider, providing OIDC and LDAP
|
|||||||
- `blueprint-argocd-oidc.yaml` — OIDC provider for ArgoCD
|
- `blueprint-argocd-oidc.yaml` — OIDC provider for ArgoCD
|
||||||
- `blueprint-freshrss-oidc.yaml` — OIDC provider for FreshRSS
|
- `blueprint-freshrss-oidc.yaml` — OIDC provider for FreshRSS
|
||||||
- `blueprint-actualbudget-proxy.yaml` — Proxy provider for Actual Budget
|
- `blueprint-actualbudget-proxy.yaml` — Proxy provider for Actual Budget
|
||||||
- `blueprint-longhorn-oidc.yaml` — OIDC provider for Longhorn (Envoy Gateway SecurityPolicy)
|
|
||||||
- `blueprint-scouting.yaml` — immich's login with Scouts Online, see docs/immich-scouts-online.md
|
|
||||||
- `blueprint-session-duration.yaml` — Session lifetime of the default authentication flow
|
- `blueprint-session-duration.yaml` — Session lifetime of the default authentication flow
|
||||||
- **NetworkPolicies** (`networkpolicy.yaml`) — the pods reach each other, DNS and the internet,
|
|
||||||
not the LAN or other namespaces; the worker and the database also reach the API server
|
|
||||||
|
|
||||||
## Session lifetime
|
## Session lifetime
|
||||||
|
|
||||||
@@ -34,23 +30,9 @@ Apache mod_auth_openidc refreshes on every request.
|
|||||||
|
|
||||||
## LDAP Outpost
|
## LDAP Outpost
|
||||||
|
|
||||||
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by
|
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by:
|
||||||
**Stalwart** (NixOS on node001), which connects via `ldap.rubenhensen.nl:389`.
|
- **SOGo-mail** (in-cluster) — connects via `ak-outpost-ldap-outpost.authentik.svc.cluster.local:3389`
|
||||||
The outpost itself talks to the server through `https://authentik.rubenhensen.nl/`.
|
- **Stalwart** (NixOS, external) — connects via `ldap.rubenhensen.nl:389`
|
||||||
|
|
||||||
Authentik's worker deploys the outpost and keeps its version equal to the
|
|
||||||
server's. After the upgrades to 2026.5 and 2026.8 it didn't follow on its
|
|
||||||
own; this rolls it over:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl -n authentik exec deploy/authentik-worker -- ak shell -c '
|
|
||||||
from authentik.outposts.models import Outpost
|
|
||||||
from authentik.outposts.tasks import outpost_controller
|
|
||||||
outpost_controller.send(Outpost.objects.get(name="LDAP outpost").pk)'
|
|
||||||
```
|
|
||||||
|
|
||||||
Upgrades go one release at a time (2026.2 → 2026.5 → 2026.8); since 2026.8
|
|
||||||
authentik refuses to skip one.
|
|
||||||
|
|
||||||
### NodePort Service
|
### NodePort Service
|
||||||
|
|
||||||
@@ -105,8 +87,3 @@ microk8s config > ~/.kube/config
|
|||||||
```
|
```
|
||||||
|
|
||||||
**Note:** This must be re-applied after MicroK8s upgrades or cert rotations, as MicroK8s may regenerate the CA without the extension. The fix needs to run on every node in the cluster.
|
**Note:** This must be re-applied after MicroK8s upgrades or cert rotations, as MicroK8s may regenerate the CA without the extension. The fix needs to run on every node in the cluster.
|
||||||
|
|
||||||
Checked 2026-10-25: the CA from this fix (`CN=microk8s-ca`, 2026-03-21, valid
|
|
||||||
until 2036) still has the extension after the refreshes to 1.34, 1.35 and
|
|
||||||
1.36. Check with
|
|
||||||
`openssl x509 -in /var/snap/microk8s/current/certs/ca.crt -noout -ext keyUsage`.
|
|
||||||
@@ -8,10 +8,6 @@ spec:
|
|||||||
# start an in-place major upgrade. CNPG filled this in at creation.
|
# start an in-place major upgrade. CNPG filled this in at creation.
|
||||||
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
|
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
|
||||||
primaryUpdateStrategy: unsupervised
|
primaryUpdateStrategy: unsupervised
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 50m
|
|
||||||
memory: 256Mi
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
initdb:
|
initdb:
|
||||||
database: authentik
|
database: authentik
|
||||||
|
|||||||
@@ -1,19 +1,5 @@
|
|||||||
# Velero Backup Testing Guide
|
# Velero Backup Testing Guide
|
||||||
|
|
||||||
**Use `restore-test.sh`.** It restores a few namespaces from a real backup
|
|
||||||
into a throwaway k3s in Docker, from a read-only copy of the bucket, and
|
|
||||||
checks that every volume comes back with the bytes it was backed up with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
backup-testing/restore-test.sh # newest daily backup; ntfy, freshrss, actual-budget
|
|
||||||
backup-testing/restore-test.sh <backup> <namespace>... # something else
|
|
||||||
```
|
|
||||||
|
|
||||||
How it works and why it copies instead of pointing Velero at the NAS is at
|
|
||||||
the top of the script. The rest of this README describes the older k3d
|
|
||||||
scripts, which predate Envoy Gateway and still expect `*.hensen.io` and
|
|
||||||
sol-auth.
|
|
||||||
|
|
||||||
This guide explains how to test your Velero backups using a temporary K3d cluster.
|
This guide explains how to test your Velero backups using a temporary K3d cluster.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|||||||
@@ -1,222 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Restore a few namespaces from the real Velero backups into a throwaway
|
|
||||||
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
|
|
||||||
# changes:
|
|
||||||
# - one backup (backups/<name>/) and the kopia repositories of the chosen
|
|
||||||
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with
|
|
||||||
# rclone into a Docker volume. A second Velero on the real bucket could
|
|
||||||
# run kopia maintenance on the same repositories as the cluster's own
|
|
||||||
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
|
|
||||||
# ReadOnly backup storage location
|
|
||||||
# - storage class longhorn becomes local-path; only core workload
|
|
||||||
# resources are restored (no routes, certificates, ExternalSecrets or
|
|
||||||
# NetworkPolicies)
|
|
||||||
# - every pod volume restore has to complete with as many bytes as its
|
|
||||||
# backup had, and the restored deployments have to become available
|
|
||||||
#
|
|
||||||
# Usage (on a machine with docker, kubectl and jq):
|
|
||||||
# backup-testing/restore-test.sh [backup] [namespace ...]
|
|
||||||
# Defaults: the newest Completed daily-backup, and ntfy freshrss
|
|
||||||
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
|
|
||||||
#
|
|
||||||
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
|
|
||||||
# into a temporary directory, and the copied backup holds every Secret of
|
|
||||||
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
|
|
||||||
# cluster (kubeconfig in the printed directory) until you run it with
|
|
||||||
# CLEANUP=1. FROM_VOLUME=<docker volume> takes the bucket from a volume
|
|
||||||
# instead of the NAS (for rehearsals; laid out as <bucket>/backups/...).
|
|
||||||
#
|
|
||||||
# Rehearsed on 2026-10-25 against a backup made in the test cluster itself,
|
|
||||||
# which is how two pitfalls showed up: Velero skips hostPath volumes, and
|
|
||||||
# k3s only provisions local PVs (which it can back up and restore) for a
|
|
||||||
# storage class with the annotation defaultVolumeType: local.
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
|
|
||||||
PORT=${PORT:-16443}
|
|
||||||
K3S=rancher/k3s:v1.34.9-k3s1
|
|
||||||
RCLONE=rclone/rclone:1.75.1
|
|
||||||
HELM=alpine/helm:3.19.0
|
|
||||||
VELERO_CHART=12.2.1
|
|
||||||
AWS_PLUGIN=velero/velero-plugin-for-aws:v1.14.4
|
|
||||||
NET=restore-test VOL=restore-test-s3
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
docker rm -f -v restore-k3s restore-s3 >/dev/null 2>&1 || true
|
|
||||||
docker volume rm "$VOL" >/dev/null 2>&1 || true
|
|
||||||
docker network rm "$NET" >/dev/null 2>&1 || true
|
|
||||||
[[ -n ${WORK:-} ]] && rm -rf "$WORK"
|
|
||||||
}
|
|
||||||
if [[ ${CLEANUP:-} == 1 ]]; then
|
|
||||||
cleanup; rm -rf "${TMPDIR:-/tmp}"/restore-test.*; echo "cleaned up"; exit 0
|
|
||||||
fi
|
|
||||||
if docker ps -a --format '{{.Names}}' | grep -qx restore-k3s; then
|
|
||||||
echo "A test cluster from an earlier run exists; run with CLEANUP=1 first." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
WORK=$(mktemp -d "${TMPDIR:-/tmp}/restore-test.XXXXXX")
|
|
||||||
chmod 700 "$WORK"
|
|
||||||
[[ ${KEEP:-} == 1 ]] || trap cleanup EXIT
|
|
||||||
|
|
||||||
pk() { kubectl --kubeconfig "$PROD_KUBECONFIG" "$@"; }
|
|
||||||
tk() { kubectl --kubeconfig "$WORK/kubeconfig" "$@"; }
|
|
||||||
# wait_for <seconds> <what> <command...>
|
|
||||||
wait_for() {
|
|
||||||
local limit=$1 what=$2; shift 2
|
|
||||||
local end=$((SECONDS + limit))
|
|
||||||
until "$@" >/dev/null 2>&1; do
|
|
||||||
if ((SECONDS > end)); then echo "timed out waiting for $what" >&2; exit 1; fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# Endpoint, region and bucket as Velero itself uses them. MinIO answers a
|
|
||||||
# request signed for the wrong region (rclone defaults to us-east-1) with a
|
|
||||||
# bare 400 on HEAD.
|
|
||||||
if [[ -n ${FROM_VOLUME:-} ]]; then
|
|
||||||
BUCKET=${BUCKET:-velero-backups}
|
|
||||||
else
|
|
||||||
bsl=$(pk -n velero get backupstoragelocations default -o json)
|
|
||||||
MINIO=${MINIO:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
|
|
||||||
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
|
|
||||||
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
|
|
||||||
fi
|
|
||||||
|
|
||||||
backup=${1:-}
|
|
||||||
shift || true
|
|
||||||
namespaces=("$@")
|
|
||||||
[[ ${#namespaces[@]} -gt 0 ]] || namespaces=(ntfy freshrss actual-budget)
|
|
||||||
if [[ -z $backup ]]; then
|
|
||||||
backup=$(pk -n velero get backups.velero.io -o json | jq -r '[.items[]
|
|
||||||
| select(.metadata.labels["velero.io/schedule-name"] == "daily-backup" and .status.phase == "Completed")]
|
|
||||||
| max_by(.status.startTimestamp).metadata.name')
|
|
||||||
fi
|
|
||||||
echo "== backup $backup, namespaces: ${namespaces[*]}"
|
|
||||||
|
|
||||||
docker network create "$NET" >/dev/null
|
|
||||||
docker volume create "$VOL" >/dev/null
|
|
||||||
if [[ -n ${FROM_VOLUME:-} ]]; then
|
|
||||||
echo "== copy from the volume $FROM_VOLUME"
|
|
||||||
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
|
|
||||||
else
|
|
||||||
echo "== copy from the NAS (read-only)"
|
|
||||||
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
|
||||||
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
|
||||||
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
|
||||||
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$MINIO" "$REGION" >> "$WORK/nas.env")
|
|
||||||
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
|
||||||
copy "backups/$backup"
|
|
||||||
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
|
||||||
rm -f "$WORK/nas.env"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "== local S3 and k3s"
|
|
||||||
docker run -d --name restore-s3 --network "$NET" -v "$VOL:/data" "$RCLONE" \
|
|
||||||
serve s3 /data --addr :8080 --auth-key test,restore-test-secret >/dev/null
|
|
||||||
s3ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" restore-s3)
|
|
||||||
docker run -d --name restore-k3s --privileged --network "$NET" -p "127.0.0.1:$PORT:6443" "$K3S" \
|
|
||||||
server --disable traefik --disable metrics-server --tls-san 127.0.0.1 --tls-san restore-k3s >/dev/null
|
|
||||||
node_ready() { docker exec restore-k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; }
|
|
||||||
wait_for 180 "k3s" node_ready
|
|
||||||
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$PORT#" > "$WORK/kubeconfig"
|
|
||||||
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed 's#https://127.0.0.1:6443#https://restore-k3s:6443#' > "$WORK/kubeconfig-net"
|
|
||||||
|
|
||||||
echo "== Velero"
|
|
||||||
cat > "$WORK/values.yaml" <<EOF
|
|
||||||
configuration:
|
|
||||||
backupStorageLocation:
|
|
||||||
- name: default
|
|
||||||
provider: aws
|
|
||||||
bucket: $BUCKET
|
|
||||||
accessMode: ReadOnly
|
|
||||||
config:
|
|
||||||
region: minio
|
|
||||||
s3ForcePathStyle: "true"
|
|
||||||
s3Url: http://$s3ip:8080
|
|
||||||
volumeSnapshotLocation: []
|
|
||||||
defaultRepoMaintainFrequency: 8760h
|
|
||||||
credentials:
|
|
||||||
secretContents:
|
|
||||||
cloud: |
|
|
||||||
[default]
|
|
||||||
aws_access_key_id=test
|
|
||||||
aws_secret_access_key=restore-test-secret
|
|
||||||
initContainers:
|
|
||||||
- name: velero-plugin-for-aws
|
|
||||||
image: $AWS_PLUGIN
|
|
||||||
volumeMounts:
|
|
||||||
- mountPath: /target
|
|
||||||
name: plugins
|
|
||||||
deployNodeAgent: true
|
|
||||||
snapshotsEnabled: false
|
|
||||||
upgradeCRDs: false
|
|
||||||
EOF
|
|
||||||
docker run --rm --network "$NET" -e HOME=/tmp -e KC="$(base64 < "$WORK/kubeconfig-net")" \
|
|
||||||
-e VALUES="$(base64 < "$WORK/values.yaml")" --entrypoint sh "$HELM" -c '
|
|
||||||
echo "$KC" | base64 -d > /tmp/kc && echo "$VALUES" | base64 -d > /tmp/v.yaml &&
|
|
||||||
helm --kubeconfig /tmp/kc install velero velero --repo https://vmware-tanzu.github.io/helm-charts \
|
|
||||||
--version '"$VELERO_CHART"' -n velero --create-namespace -f /tmp/v.yaml >/dev/null'
|
|
||||||
tk apply -f - >/dev/null <<EOF
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: change-storage-class
|
|
||||||
namespace: velero
|
|
||||||
labels:
|
|
||||||
velero.io/plugin-config: ""
|
|
||||||
velero.io/change-storage-class: RestoreItemAction
|
|
||||||
data:
|
|
||||||
longhorn: local-path
|
|
||||||
EOF
|
|
||||||
wait_for 300 "Velero to sync $backup" tk -n velero get backups.velero.io "$backup"
|
|
||||||
echo " Velero sees $backup"
|
|
||||||
|
|
||||||
echo "== restore"
|
|
||||||
tk apply -f - >/dev/null <<EOF
|
|
||||||
apiVersion: velero.io/v1
|
|
||||||
kind: Restore
|
|
||||||
metadata:
|
|
||||||
name: restore-test
|
|
||||||
namespace: velero
|
|
||||||
spec:
|
|
||||||
backupName: $backup
|
|
||||||
includedNamespaces: [$(IFS=,; echo "${namespaces[*]}")]
|
|
||||||
# PersistentVolumes have to be included: only then does Velero clear the
|
|
||||||
# PVC's volumeName. It doesn't restore a PV whose data comes from kopia;
|
|
||||||
# k3s provisions a new one.
|
|
||||||
includedResources: [namespaces, persistentvolumeclaims, persistentvolumes, pods, deployments,
|
|
||||||
replicasets, statefulsets, services, configmaps, secrets, serviceaccounts]
|
|
||||||
restorePVs: true
|
|
||||||
EOF
|
|
||||||
restore_done() { [[ $(tk -n velero get restore restore-test -o jsonpath='{.status.phase}') =~ ^(Completed|PartiallyFailed|Failed|FailedValidation)$ ]]; }
|
|
||||||
wait_for 1800 "the restore" restore_done
|
|
||||||
phase=$(tk -n velero get restore restore-test -o jsonpath='{.status.phase}')
|
|
||||||
echo " restore: $phase"
|
|
||||||
|
|
||||||
echo "== check"
|
|
||||||
failed=0
|
|
||||||
expected=$(docker run --rm -v "$VOL:/data" alpine:3.22 zcat "/data/$BUCKET/backups/$backup/$backup-podvolumebackups.json.gz" |
|
|
||||||
jq -r --arg ns "$(IFS=,; echo "${namespaces[*]}")" '.[] | select(.spec.pod.namespace as $p | $ns | split(",") | index($p))
|
|
||||||
| "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0)"' | sort -u)
|
|
||||||
restored=$(tk -n velero get podvolumerestores.velero.io -o json |
|
|
||||||
jq -r '.items[] | "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0) \(.status.phase)"' | sort -u)
|
|
||||||
while read -r vol bytes; do
|
|
||||||
[[ -z $vol ]] && continue
|
|
||||||
got=$(awk -v v="$vol" '$1 == v {print $2, $3}' <<<"$restored")
|
|
||||||
if [[ $got == "$bytes Completed" ]]; then
|
|
||||||
echo " ok $vol ($bytes bytes)"
|
|
||||||
else
|
|
||||||
echo " FAILED $vol: backed up $bytes bytes, restore: ${got:-none}"; failed=1
|
|
||||||
fi
|
|
||||||
done <<<"$expected"
|
|
||||||
for ns in "${namespaces[@]}"; do
|
|
||||||
if tk -n "$ns" wait --for=condition=Available deployment --all --timeout=300s >/dev/null 2>&1; then
|
|
||||||
echo " ok $ns: deployments available"
|
|
||||||
else
|
|
||||||
echo " FAILED $ns: deployments not available"; tk -n "$ns" get pods; failed=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
[[ $phase == Completed ]] || failed=1
|
|
||||||
[[ ${KEEP:-} == 1 ]] && echo "Kept; kubeconfig: $WORK/kubeconfig. Remove with CLEANUP=1 $0"
|
|
||||||
if [[ $failed == 0 ]]; then echo "== restore test passed"; else echo "== restore test FAILED"; exit 1; fi
|
|
||||||
@@ -123,13 +123,15 @@ D. auth.scoutingmbg.nl (2026-10-25)
|
|||||||
|
|
||||||
## Open points
|
## Open points
|
||||||
|
|
||||||
- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
|
- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
|
||||||
scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
|
foto.scoutingmbg.nl from a CNAME to `mbg.hensen.io` (the home IP) to a
|
||||||
all the records it moved, and node001 tunnels all four home. ns1 lagged
|
CNAME to `rubenhensen.nl`, which is node001 on Hetzner. Only ns2 has it
|
||||||
behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
|
(serial 2026102471); ns1 is stuck on serial 2026060640 from June and
|
||||||
uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
|
still answers `mbg.hensen.io`. node001 now tunnels foto.scoutingmbg.nl to
|
||||||
`a` now means node001, which is fine: the WordPress site sends no mail
|
the home cluster like its other hosts, so both answers work. Keep
|
||||||
and the group's mail goes through Google (`mx`)
|
hensen.io until ns1 has caught up; ask the DNS host to fix ns1, and
|
||||||
|
decide whether foto should stay behind node001 or point straight home
|
||||||
|
(CNAME `scoutingmbg.nl`)
|
||||||
- Public shared links become public for real once sol-auth is gone.
|
- Public shared links become public for real once sol-auth is gone.
|
||||||
- Photos leiding back up from their phones stay private until they add
|
- Photos leiding back up from their phones stay private until they add
|
||||||
them to a shared album.
|
them to a shared album.
|
||||||
@@ -38,15 +38,14 @@ between them live.
|
|||||||
|
|
||||||
## Before: shrink what has to move
|
## Before: shrink what has to move
|
||||||
|
|
||||||
- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
- [ ] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
||||||
snapshots from before October 2026, nearly all on immich-claim
|
snapshots from before October 2026, nearly all on immich-claim
|
||||||
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
|
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
|
||||||
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
|
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
|
||||||
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
||||||
don't depend on them. Deleting a snapshot merges it into the next one,
|
don't depend on them. Deleting a snapshot merges it into the next one,
|
||||||
which writes a lot, so do this in a quiet hour, one volume at a time.
|
which writes a lot, so do this in a quiet hour, one volume at a time.
|
||||||
Happened on 2026-10-24, from a Claude Code session at Ruben's request
|
Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
|
||||||
(the 45 snapshots from before October 2026, small volumes first): immich-claim's
|
|
||||||
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
||||||
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
||||||
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
||||||
@@ -55,16 +54,6 @@ between them live.
|
|||||||
on immich-claim is one 233 GiB snapshot (`system-backup-0b7f8c6047294b36`,
|
on immich-claim is one 233 GiB snapshot (`system-backup-0b7f8c6047294b36`,
|
||||||
from the Longhorn system backup of 2026-10-22) that everything merged
|
from the Longhorn system backup of 2026-10-22) that everything merged
|
||||||
into; 148 snapshots hold 273 GiB across all volumes.
|
into; 148 snapshots hold 273 GiB across all volumes.
|
||||||
- [ ] Delete the snapshots the Longhorn system backups left behind: 116
|
|
||||||
`system-backup-*` snapshots (one per volume for each system backup
|
|
||||||
before an upgrade step on 2026-10-22 and 23) hold 258 GiB. The system
|
|
||||||
backups on the NAS don't need them. The small volumes are cheap; on
|
|
||||||
immich-claim the 233 GiB one is the base the rest merged into, so
|
|
||||||
deleting it means hours of merging like on 2026-10-24: one night, or
|
|
||||||
after the move to the NVMe drive. Then trim (next item); deleted files
|
|
||||||
only free space once both are done. Next time, take system backups with
|
|
||||||
the volume backup policy `disabled` when the nightly backups are fresh,
|
|
||||||
so they create no snapshots
|
|
||||||
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
|
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
|
||||||
so blocks of deleted files are released.
|
so blocks of deleted files are released.
|
||||||
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
|
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
|
||||||
|
|||||||
+28
-208
@@ -8,15 +8,11 @@ bottom.
|
|||||||
|
|
||||||
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
||||||
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
||||||
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
35 Argo CD apps, all Synced/Healthy; every Longhorn volume has one
|
||||||
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
|
replica. Backups go to the NAS (192.168.1.99): Longhorn to NFS at 00:00
|
||||||
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
|
(7 kept), Velero to MinIO at 01:00 (daily kept 30 days, weekly 90 days).
|
||||||
01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
|
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
||||||
and s2 (home) watch each other and alert through ntfy; s2 also gets an
|
through ntfy. node001 runs mail and tunnels the public hosts home.
|
||||||
hourly report from inside the cluster on the backups, Vault's seal, the
|
|
||||||
Argo CD apps and the CNPG clusters.
|
|
||||||
Every app namespace has NetworkPolicies, and CI renders and validates all
|
|
||||||
apps on every PR. node001 runs mail and tunnels the public hosts home.
|
|
||||||
|
|
||||||
## Follow-ups from this week
|
## Follow-ups from this week
|
||||||
|
|
||||||
@@ -34,9 +30,8 @@ docs/longhorn-second-ssd.md:
|
|||||||
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
||||||
(`backend-*.tar`, `dqlite-pre-*`)
|
(`backend-*.tar`, `dqlite-pre-*`)
|
||||||
- [ ] Log in to immich as a jeugdlid (quota 0)
|
- [ ] Log in to immich as a jeugdlid (quota 0)
|
||||||
- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
|
- [ ] Back to 1.36/stable once it is at v1.36.5 or later. The snap is held,
|
||||||
2026-10-25). The snap is held, so microk8s gets no updates at all until
|
so microk8s gets no updates at all until someone refreshes it
|
||||||
someone refreshes it
|
|
||||||
- [ ] Second SSD, docs/longhorn-second-ssd.md
|
- [ ] Second SSD, docs/longhorn-second-ssd.md
|
||||||
- [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
|
- [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
|
||||||
|
|
||||||
@@ -50,9 +45,8 @@ group's ns2: ns1 is stuck on an old serial and still answers
|
|||||||
mbg.hensen.io (docs/immich-scouts-online.md).
|
mbg.hensen.io (docs/immich-scouts-online.md).
|
||||||
|
|
||||||
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
||||||
- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
|
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
|
||||||
`rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
|
rubenhensen.nl too
|
||||||
cluster uses hensen.io anymore. hensen.io can expire
|
|
||||||
|
|
||||||
### 2. Off-site backups
|
### 2. Off-site backups
|
||||||
|
|
||||||
@@ -83,34 +77,7 @@ friend.
|
|||||||
the NAS
|
the NAS
|
||||||
- [ ] Decide which copy of the immich library goes off-site (see 5)
|
- [ ] Decide which copy of the immich library goes off-site (see 5)
|
||||||
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
||||||
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
|
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth
|
||||||
Planned on 2026-10-25, not run: a second Velero on the real bucket is
|
|
||||||
risky even with a ReadOnly location, because kopia could then run
|
|
||||||
maintenance on the same repositories from two clusters. Instead: copy
|
|
||||||
one backup (`backups/<name>/`) and the kopia repositories of a few small
|
|
||||||
namespaces (`kopia/{gatus,ntfy,freshrss}/`) read-only from the NAS with
|
|
||||||
rclone into a local folder, serve that with `rclone serve s3`, and
|
|
||||||
restore into a throwaway k3s in Docker (Velero 1.18.2, node agent on,
|
|
||||||
storage class longhorn mapped to local-path, without httproutes,
|
|
||||||
certificates, referencegrants and networkpolicies). It needs the MinIO
|
|
||||||
key outside the cluster, which wants your go-ahead; better still, a
|
|
||||||
MinIO user with read-only access to `velero-backups`.
|
|
||||||
Built as `backup-testing/restore-test.sh` (2026-10-25) and rehearsed
|
|
||||||
end to end against a backup made inside the test cluster, with a
|
|
||||||
ReadOnly location: data restored byte for byte (same sha256), deployment
|
|
||||||
available, everything removed afterwards. Two pitfalls found on the way
|
|
||||||
are in the script's header. Not yet run against the real backups:
|
|
||||||
- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
|
|
||||||
daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
|
|
||||||
volume came back with its backed-up byte count (actual-budget 8.4 MB,
|
|
||||||
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
|
|
||||||
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
|
|
||||||
MinIO runs region `minio`; the script now reads endpoint, region and
|
|
||||||
bucket from Velero's location. Copying freshrss's kopia repository took
|
|
||||||
14 of the ~17 minutes
|
|
||||||
- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
|
|
||||||
the cluster itself: the Mac (launchd) or node001, ideally with a
|
|
||||||
read-only MinIO user
|
|
||||||
|
|
||||||
### 3. authentik 2026.2.1 → 2026.8.5
|
### 3. authentik 2026.2.1 → 2026.8.5
|
||||||
|
|
||||||
@@ -138,13 +105,10 @@ April and May.
|
|||||||
- [x] Log in once through each app: immich through Scouts Online, Argo CD,
|
- [x] Log in once through each app: immich through Scouts Online, Argo CD,
|
||||||
Vault, Longhorn, FreshRSS, Actual, mail
|
Vault, Longhorn, FreshRSS, Actual, mail
|
||||||
- [ ] After a week without problems: delete the dump and the snapshot
|
- [ ] After a week without problems: delete the dump and the snapshot
|
||||||
- [x] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`), required
|
- [ ] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`) becomes
|
||||||
from 2026.11: already `https://authentik.rubenhensen.nl`, filled in by
|
required in 2026.11. authentik answers on authentik.rubenhensen.nl and
|
||||||
the 2026.8 migration from the embedded outpost's host. In 2026.8.5 it is
|
auth.scoutingmbg.nl (immich's issuer), so find out first what the
|
||||||
only stored; on main it fills itself in the same way and sets the
|
setting does with two hostnames
|
||||||
embedded outpost's host from it. OIDC issuers still come from the
|
|
||||||
request's host, so immich's issuer on auth.scoutingmbg.nl is unaffected
|
|
||||||
(checked 2026-10-25 in the source)
|
|
||||||
|
|
||||||
### 4. Vault 1.17.2 → 2.2.1
|
### 4. Vault 1.17.2 → 2.2.1
|
||||||
|
|
||||||
@@ -165,8 +129,8 @@ unseals it.
|
|||||||
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
||||||
doesn't exist, so the OIDC mount's listing visibility was never set.
|
doesn't exist, so the OIDC mount's listing visibility was never set.
|
||||||
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
||||||
- [x] There is one unseal key (1 share, threshold 1), kept safe outside
|
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
|
||||||
this cluster (2026-10-25); without it the data in Vault is lost
|
places outside this cluster; without it the data in Vault is lost
|
||||||
- [x] Unsealing is in the SSD install steps
|
- [x] Unsealing is in the SSD install steps
|
||||||
- [ ] After a week without problems: delete the three snapshots
|
- [ ] After a week without problems: delete the three snapshots
|
||||||
|
|
||||||
@@ -215,21 +179,15 @@ until 2026-10-21 without anyone noticing.
|
|||||||
|
|
||||||
- [x] A stopgap for the worst blind spots (2026-10-25,
|
- [x] A stopgap for the worst blind spots (2026-10-25,
|
||||||
`gatus/backup-check.yaml`): an hourly CronJob reads the API and pushes
|
`gatus/backup-check.yaml`): an hourly CronJob reads the API and pushes
|
||||||
six external endpoints to s2 (group "cluster"): the newest daily and
|
four external endpoints to s2 (group "cluster"): the newest daily and
|
||||||
weekly Velero backup Completed and recent, every attached Longhorn
|
weekly Velero backup Completed and recent, every attached Longhorn
|
||||||
volume healthy with a backup younger than 26h, no Vault pod sealed,
|
volume healthy with a backup younger than 26h, and no Vault pod sealed.
|
||||||
every Argo CD app Synced and Healthy, every CNPG cluster healthy.
|
|
||||||
Gatus alerts through ntfy after two failed reports, or when no report
|
Gatus alerts through ntfy after two failed reports, or when no report
|
||||||
arrives for 3 hours. The token comes from an ESO Password generator.
|
arrives for 3 hours. The token comes from an ESO Password generator.
|
||||||
Tested with made-up data for each failure (a Failed or old backup, a
|
Tested with made-up data for each failure (a Failed or old backup, a
|
||||||
degraded or unbacked-up volume, a sealed pod)
|
degraded or unbacked-up volume, a sealed pod)
|
||||||
- [x] metrics-server (2026-10-25, `apps/templates/metrics-server-helm.yaml`):
|
|
||||||
`kubectl top` works; it keeps its data in memory, so it adds no disk
|
|
||||||
writes. The first look: immich v3 checks file checksums every night from
|
|
||||||
03:00 UTC for up to an hour (IntegrityService), reading ~100 MB/s from
|
|
||||||
sda; it doesn't overlap the backups at 00:00 and 01:00
|
|
||||||
- [ ] After the second SSD (a time series database writes all the time
|
- [ ] After the second SSD (a time series database writes all the time
|
||||||
and would add to today's stalls): Prometheus or
|
and would add to today's stalls): metrics-server, then Prometheus or
|
||||||
VictoriaMetrics with Grafana, alerting to ntfy. Loki later
|
VictoriaMetrics with Grafana, alerting to ntfy. Loki later
|
||||||
- [ ] Alerts worth having: disk latency and fill, lease losses and
|
- [ ] Alerts worth having: disk latency and fill, lease losses and
|
||||||
controller restarts, backups that don't complete, Longhorn volumes that
|
controller restarts, backups that don't complete, Longhorn volumes that
|
||||||
@@ -277,48 +235,9 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
||||||
default) still have none; `default` can reach the LAN and everything
|
default) still have none; `default` can reach the LAN and everything
|
||||||
without a policy
|
without a policy
|
||||||
- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
|
- [ ] Requests and memory limits, the big ones first (immich, authentik,
|
||||||
had none except the Vault server, from `kubectl top` every 5 minutes
|
Argo CD's repo-server)
|
||||||
from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
|
- [ ] Probes
|
||||||
floor). Rolled out in about eight minutes; both CNPG instances
|
|
||||||
restarted and came back healthy, the OIDC endpoints, the logins and
|
|
||||||
immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
|
|
||||||
1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
|
|
||||||
requested. Still without requests: Longhorn (22 containers, its chart),
|
|
||||||
the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
|
|
||||||
kube-system
|
|
||||||
- [ ] Memory limits, once there are daytime numbers: the night ones miss
|
|
||||||
immich's machine learning at work (it idles at 234Mi)
|
|
||||||
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
|
|
||||||
2026-10-25, its third restart: a crawler asked for 88 pages and assets
|
|
||||||
within a few seconds (normally ~6 a minute), Apache started a PHP worker
|
|
||||||
for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
|
|
||||||
the container went past its 1Gi limit. Unrelated to the
|
|
||||||
NetworkPolicies. Now capped at 12 workers through the chart's
|
|
||||||
`apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
|
|
||||||
with `apache2ctl -t` in the same image first. Replaying the burst (90
|
|
||||||
page requests, 40 at a time): all 200, at most 13 Apache processes and
|
|
||||||
572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
|
|
||||||
peak) instead of a crash
|
|
||||||
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
|
|
||||||
cluster between Envoy and WordPress that keeps anonymous GETs for a few
|
|
||||||
minutes and passes anything with a WordPress cookie through. A page
|
|
||||||
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
|
|
||||||
waiting in a burst is queueing; the cap alone already prevents the crash
|
|
||||||
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
|
|
||||||
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
|
|
||||||
little here, `wordpress-importer` is usually for a one-off import, and
|
|
||||||
`code-snippets` runs PHP from the dashboard
|
|
||||||
- [ ] Rate limiting per visitor in Envoy needs the client's address, which
|
|
||||||
node001 loses: it passes TLS through, so every request arrives from
|
|
||||||
46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
|
|
||||||
(ClientTrafficPolicy `enableProxyProtocol`)
|
|
||||||
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
|
|
||||||
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
|
|
||||||
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
|
|
||||||
SSD's stalls. Still without probes: authentik's LDAP outpost (deployed
|
|
||||||
by authentik), the ESO controller, dex and Argo CD's redis (upstream
|
|
||||||
manifests)
|
|
||||||
- [ ] Non-root where the image allows it
|
- [ ] Non-root where the image allows it
|
||||||
|
|
||||||
### 8. Renovate and CI
|
### 8. Renovate and CI
|
||||||
@@ -326,58 +245,9 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
10 Renovate PRs are open, the oldest from February, and nothing checks a
|
10 Renovate PRs are open, the oldest from February, and nothing checks a
|
||||||
PR before Argo CD applies it.
|
PR before Argo CD applies it.
|
||||||
|
|
||||||
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
|
- [ ] A GitHub Actions workflow that renders the Helm apps and runs
|
||||||
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
|
kubeconform on every PR, with conftest for this repo's rules
|
||||||
does (the apps chart, every Application's chart with its values, value
|
- [ ] Automerge patch versions of low-risk apps once that runs
|
||||||
files and parameters, kustomizations, plain folders, and
|
|
||||||
external-snapshotter cloned with its patches) and validates the result
|
|
||||||
with kubeconform (strict, CRD schemas from the datree catalog), plus a
|
|
||||||
duplicate-key check on every YAML file. Helm 4.2.1 like Argo CD 3.5,
|
|
||||||
versions pinned and checksummed, kept current by Renovate through the
|
|
||||||
`# renovate:` comments. Tested by planting a typo in a Deployment, a
|
|
||||||
chart version that doesn't exist and a value the immich chart's schema
|
|
||||||
rejects; all three fail the run. Locally:
|
|
||||||
`docker run --rm -v $PWD:/repo:ro <image with helm, kustomize, kubeconform, PyYAML> python /repo/tools/ci/validate.py`
|
|
||||||
- It found `syncPolicy.automated` twice in
|
|
||||||
`apps/templates/snapshot-controller-helm.yaml` (once with `prune: true`,
|
|
||||||
added 2025-03-03); Argo CD kept the last one, without prune. Now listed
|
|
||||||
once, as it ran. Prune was probably meant; turn it on deliberately if so
|
|
||||||
- [x] This repo's own rules, in `tools/ci/validate.py` rather than
|
|
||||||
conftest (2026-10-25), on what the repo's own folders render: no Secret
|
|
||||||
with data in git, and every image has a tag other than latest (both
|
|
||||||
fail the run; tested with a planted Secret and an untagged image), plus
|
|
||||||
PR annotations for containers without requests or probes. freshrss's
|
|
||||||
`alpine/git` init container had no tag; now `v2.52.0`
|
|
||||||
- [ ] Automerge patch versions of low-risk apps now that CI runs
|
|
||||||
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
|
|
||||||
`argocd` manager and a custom `github-tags` regex both matched. The
|
|
||||||
regex is gone; Renovate should close #15 by itself
|
|
||||||
- [x] Renovate only looked at six folders for images and only at
|
|
||||||
apps/templates for Applications, so it never offered updates for the
|
|
||||||
vault, immich, velero, mariadb-operator and actualbudget charts, or for
|
|
||||||
images in freshrss, mbgwp, dns, vault and immich, or for CNPG's
|
|
||||||
`imageName` (authentik still runs PostgreSQL 17.4). Both managers now
|
|
||||||
cover every app folder, a regex manager reads `imageName` in
|
|
||||||
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
|
|
||||||
(those need pg_upgrade or a dump). Expect a batch of new PRs
|
|
||||||
(2026-10-25; `renovate-config-validator` passes). They only come once
|
|
||||||
fewer than 10 Renovate PRs are open (`config:recommended`'s
|
|
||||||
`prConcurrentLimit`); until then they wait under "Rate-Limited" on the
|
|
||||||
Dependency Dashboard (#2)
|
|
||||||
- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
|
|
||||||
v1.21.3 (security release; all three pods on v1.21.3), #24
|
|
||||||
external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
|
|
||||||
v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
|
|
||||||
0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
|
|
||||||
ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
|
|
||||||
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
|
|
||||||
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
|
|
||||||
its own beforehand, the nightly ones are the way back)
|
|
||||||
- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
|
|
||||||
manual rebase (#33 had added values on the next line, which git counts
|
|
||||||
as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
|
|
||||||
ExternalSecrets refreshed on a force-sync; the 25th is the generated
|
|
||||||
`gatus/backup-check-token`, which never refreshes by design
|
|
||||||
|
|
||||||
### 9. One node
|
### 9. One node
|
||||||
|
|
||||||
@@ -393,12 +263,6 @@ step after that, if downtime starts to matter.
|
|||||||
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
||||||
completed kopia maintenance jobs of repositories that no longer exist
|
completed kopia maintenance jobs of repositories that no longer exist
|
||||||
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
||||||
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
|
|
||||||
volumes over the file paths `data/config.custom.php` and
|
|
||||||
`data/config-user.custom.php`, so FreshRSS sees two empty directories
|
|
||||||
there and the volumes hold nothing (0 bytes in every backup). Drop both
|
|
||||||
mounts and the two PVCs, or make them ConfigMap files if custom config
|
|
||||||
is wanted
|
|
||||||
- [ ] `kubectl delete namespace secrets`, empty since its app went
|
- [ ] `kubectl delete namespace secrets`, empty since its app went
|
||||||
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
|
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
|
||||||
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
|
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
|
||||||
@@ -421,53 +285,9 @@ step after that, if downtime starts to matter.
|
|||||||
- [ ] opti1, with sudo: `/root/backend-*` and `/root/dqlite-pre-*` (from
|
- [ ] opti1, with sudo: `/root/backend-*` and `/root/dqlite-pre-*` (from
|
||||||
before 1.33, 1.34 and the VACUUM; 1.34 migrated the schema, so none of
|
before 1.33, 1.34 and the VACUUM; 1.34 migrated the schema, so none of
|
||||||
them fits 1.36) and `~/containerd.bak` (root-owned, from 2025-02)
|
them fits 1.36) and `~/containerd.bak` (root-owned, from 2025-02)
|
||||||
- [ ] `rm -r ~/cadir` on opti1: a CA key and certificate made four minutes
|
- [ ] `~/cadir` on opti1 holds a CA key and certificate from 2026-03-21
|
||||||
before the real keyUsage fix on 2026-03-21 (OpenSSL's default subject
|
that are not microk8s's current CA. Find out whether anything still
|
||||||
"Internet Widgits Pty Ltd", valid for a year) and never installed;
|
trusts it; if not, delete it
|
||||||
microk8s's CA is the `CN=microk8s-ca` one from 19:16 that day, valid
|
|
||||||
until 2036
|
|
||||||
|
|
||||||
### 11. Known vulnerabilities
|
|
||||||
|
|
||||||
A one-off scan on 2026-10-25 of all 52 images running in the cluster
|
|
||||||
(Trivy 0.75.0, remote, linux/amd64, HIGH and CRITICAL only). Counts are
|
|
||||||
critical / high:
|
|
||||||
|
|
||||||
| Image | Crit | High | What to do |
|
|
||||||
|---|---|---|---|
|
|
||||||
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; goes with the cleanup from 2026-10-27 |
|
|
||||||
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. Renovate now offers 17.7; a `17.x-standard-trixie` image is more current |
|
|
||||||
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
|
||||||
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
|
||||||
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
|
||||||
| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
|
|
||||||
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
|
|
||||||
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
|
|
||||||
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
|
|
||||||
| coredns/coredns:1.10.1 | 5 | 70 | see below |
|
|
||||||
| dexidp/dex:v2.45.1, argocd v3.5.6, redis 8.2.3 | 4, 1, 2 | | come with Argo CD |
|
|
||||||
| mariadb:11.4.5 | 4 | 65 | the newest 11.4 patch; Renovate now covers it |
|
|
||||||
| calico v3.29.3 | 2-3 | ~50 | comes with microk8s |
|
|
||||||
| goauthentik ldap / server 2026.8.5 | 3, 1 | 74, 65 | the next authentik patch |
|
|
||||||
|
|
||||||
Nothing critical in Envoy, Vault, Velero, cert-manager, ESO, ntfy, gatus or
|
|
||||||
metrics-server.
|
|
||||||
|
|
||||||
- [ ] 16 volumes still have their engine in Longhorn 1.7.3's instance
|
|
||||||
manager (`instance-manager-07d66a5…`, running since March): vault ×3,
|
|
||||||
authentik-postgres, immich-postgres (PG14), mbgwp (mariadb and
|
|
||||||
wordpress), gitea, freshrss ×3, ntfy, gatus, serpbear, bolletjestrui,
|
|
||||||
actual-budget. On one node a pod restart never detaches the volume (the
|
|
||||||
new pod lands on the same node), so only a real detach moves the engine
|
|
||||||
to 1.13's instance manager: scale to 0, wait until Longhorn shows the
|
|
||||||
volume detached, scale back (Argo CD's selfHeal undoes a scale-down, so
|
|
||||||
pause it for the app). Vault comes back sealed; CNPG clusters want
|
|
||||||
`cnpg.io/hibernation=on` instead of a scale-down. The old instance
|
|
||||||
manager goes away by itself once it serves nothing
|
|
||||||
- [ ] CoreDNS runs 1.10.1 from when the addon was enabled; the addon
|
|
||||||
repository has 1.14.3. `sudo microk8s disable dns && sudo microk8s enable dns:1.1.1.1`
|
|
||||||
(a short DNS outage for the whole cluster; check the Corefile after)
|
|
||||||
- [ ] The Trivy Operator for scans that keep running (on the wishlist)
|
|
||||||
|
|
||||||
## wishlist.txt (2026-01-25)
|
## wishlist.txt (2026-01-25)
|
||||||
|
|
||||||
|
|||||||
@@ -282,13 +282,6 @@ whatever time of day the hourly schedule had drifted to.
|
|||||||
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
||||||
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
||||||
their leases at 01:31
|
their leases at 01:31
|
||||||
Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
|
|
||||||
11:30, leaving out the CNPG range cancels): all 67 minutes with errors
|
|
||||||
had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
|
|
||||||
"full" pressure at 45-78%, against a median minute peak of 24 ms. The
|
|
||||||
worst minute after the merges was the rollout of eight merged PRs at
|
|
||||||
2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
|
|
||||||
CNPG and Envoy Gateway held their 60s leases through all of it
|
|
||||||
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
||||||
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
||||||
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ spec:
|
|||||||
io.kompose.service: freshrss
|
io.kompose.service: freshrss
|
||||||
spec:
|
spec:
|
||||||
initContainers:
|
initContainers:
|
||||||
- image: alpine/git:v2.52.0
|
- image: alpine/git
|
||||||
name: git
|
name: git
|
||||||
command:
|
command:
|
||||||
- git
|
- git
|
||||||
@@ -79,23 +79,6 @@ spec:
|
|||||||
ports:
|
ports:
|
||||||
- containerPort: 80
|
- containerPort: 80
|
||||||
protocol: TCP
|
protocol: TCP
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 50m
|
|
||||||
memory: 176Mi
|
|
||||||
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
|
|
||||||
# browser.
|
|
||||||
readinessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 80
|
|
||||||
periodSeconds: 10
|
|
||||||
livenessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 80
|
|
||||||
initialDelaySeconds: 60
|
|
||||||
periodSeconds: 30
|
|
||||||
timeoutSeconds: 10
|
|
||||||
failureThreshold: 5
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /var/www/FreshRSS/data
|
- mountPath: /var/www/FreshRSS/data
|
||||||
name: data
|
name: data
|
||||||
|
|||||||
+1
-29
@@ -11,8 +11,6 @@
|
|||||||
# younger than 26h (detached volumes are skipped; the recurring job skips
|
# younger than 26h (detached volumes are skipped; the recurring job skips
|
||||||
# them too)
|
# them too)
|
||||||
# - vault-unsealed: all Vault pods report vault-sealed=false
|
# - vault-unsealed: all Vault pods report vault-sealed=false
|
||||||
# - argocd-apps: every Argo CD application is Synced and Healthy
|
|
||||||
# - postgres: every CloudNativePG cluster is in a healthy state
|
|
||||||
#
|
#
|
||||||
# The token is generated by ESO and never leaves the cluster; Gatus reads
|
# The token is generated by ESO and never leaves the cluster; Gatus reads
|
||||||
# it from the same Secret.
|
# it from the same Secret.
|
||||||
@@ -65,12 +63,6 @@ rules:
|
|||||||
- apiGroups: [""]
|
- apiGroups: [""]
|
||||||
resources: ["pods"]
|
resources: ["pods"]
|
||||||
verbs: ["list"]
|
verbs: ["list"]
|
||||||
- apiGroups: ["argoproj.io"]
|
|
||||||
resources: ["applications"]
|
|
||||||
verbs: ["list"]
|
|
||||||
- apiGroups: ["postgresql.cnpg.io"]
|
|
||||||
resources: ["clusters"]
|
|
||||||
verbs: ["list"]
|
|
||||||
---
|
---
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
kind: ClusterRoleBinding
|
kind: ClusterRoleBinding
|
||||||
@@ -178,24 +170,6 @@ data:
|
|||||||
return f"sealed: {', '.join(sealed)}" if sealed else None
|
return f"sealed: {', '.join(sealed)}" if sealed else None
|
||||||
|
|
||||||
|
|
||||||
def argocd():
|
|
||||||
bad = []
|
|
||||||
for a in items("/apis/argoproj.io/v1alpha1/namespaces/argocd/applications"):
|
|
||||||
s = a.get("status", {})
|
|
||||||
sync = s.get("sync", {}).get("status")
|
|
||||||
health = s.get("health", {}).get("status")
|
|
||||||
if sync != "Synced" or health != "Healthy":
|
|
||||||
bad.append(f"{a['metadata']['name']} {sync}/{health}")
|
|
||||||
return "; ".join(bad) or None
|
|
||||||
|
|
||||||
|
|
||||||
def postgres():
|
|
||||||
bad = [f"{c['metadata']['namespace']}/{c['metadata']['name']}: {c.get('status', {}).get('phase')}"
|
|
||||||
for c in items("/apis/postgresql.cnpg.io/v1/clusters")
|
|
||||||
if c.get("status", {}).get("phase") != "Cluster in healthy state"]
|
|
||||||
return "; ".join(bad) or None
|
|
||||||
|
|
||||||
|
|
||||||
def push(key, error):
|
def push(key, error):
|
||||||
query = {"success": "false" if error else "true"}
|
query = {"success": "false" if error else "true"}
|
||||||
if error:
|
if error:
|
||||||
@@ -211,8 +185,6 @@ data:
|
|||||||
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
|
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
|
||||||
"cluster_longhorn": lambda: longhorn(26),
|
"cluster_longhorn": lambda: longhorn(26),
|
||||||
"cluster_vault-unsealed": vault,
|
"cluster_vault-unsealed": vault,
|
||||||
"cluster_argocd-apps": argocd,
|
|
||||||
"cluster_postgres": postgres,
|
|
||||||
}
|
}
|
||||||
failed_pushes = 0
|
failed_pushes = 0
|
||||||
for key, check in checks.items():
|
for key, check in checks.items():
|
||||||
@@ -256,7 +228,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: check
|
- name: check
|
||||||
image: python:3.15.1-alpine3.24
|
image: python:3.13.16-alpine3.24
|
||||||
command: ["python", "/script/check.py"]
|
command: ["python", "/script/check.py"]
|
||||||
env:
|
env:
|
||||||
- name: PYTHONDONTWRITEBYTECODE
|
- name: PYTHONDONTWRITEBYTECODE
|
||||||
|
|||||||
@@ -169,15 +169,3 @@ external-endpoints:
|
|||||||
heartbeat:
|
heartbeat:
|
||||||
interval: 3h
|
interval: 3h
|
||||||
alerts: *cluster-alert
|
alerts: *cluster-alert
|
||||||
- name: argocd-apps
|
|
||||||
group: cluster
|
|
||||||
token: ${BACKUP_CHECK_TOKEN}
|
|
||||||
heartbeat:
|
|
||||||
interval: 3h
|
|
||||||
alerts: *cluster-alert
|
|
||||||
- name: postgres
|
|
||||||
group: cluster
|
|
||||||
token: ${BACKUP_CHECK_TOKEN}
|
|
||||||
heartbeat:
|
|
||||||
interval: 3h
|
|
||||||
alerts: *cluster-alert
|
|
||||||
@@ -20,7 +20,7 @@ spec:
|
|||||||
fsGroup: 1000
|
fsGroup: 1000
|
||||||
containers:
|
containers:
|
||||||
- name: gitea
|
- name: gitea
|
||||||
image: gitea/gitea:28.2.0
|
image: gitea/gitea:1.27.3
|
||||||
env:
|
env:
|
||||||
- name: USER_UID
|
- name: USER_UID
|
||||||
value: "1000"
|
value: "1000"
|
||||||
@@ -50,25 +50,6 @@ spec:
|
|||||||
- containerPort: 3000
|
- containerPort: 3000
|
||||||
name: http
|
name: http
|
||||||
protocol: TCP
|
protocol: TCP
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 20m
|
|
||||||
memory: 336Mi
|
|
||||||
# /api/healthz checks the database and the cache.
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /api/healthz
|
|
||||||
port: http
|
|
||||||
periodSeconds: 10
|
|
||||||
timeoutSeconds: 5
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /api/healthz
|
|
||||||
port: http
|
|
||||||
initialDelaySeconds: 60
|
|
||||||
periodSeconds: 30
|
|
||||||
timeoutSeconds: 10
|
|
||||||
failureThreshold: 5
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /data
|
- mountPath: /data
|
||||||
name: data
|
name: data
|
||||||
|
|||||||
+3
-28
@@ -63,10 +63,6 @@ spec:
|
|||||||
main:
|
main:
|
||||||
containers:
|
containers:
|
||||||
main:
|
main:
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 64Mi
|
|
||||||
image:
|
image:
|
||||||
repository: docker.io/valkey/valkey
|
repository: docker.io/valkey/valkey
|
||||||
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
||||||
@@ -89,10 +85,6 @@ spec:
|
|||||||
image:
|
image:
|
||||||
repository: ghcr.io/immich-app/immich-server
|
repository: ghcr.io/immich-app/immich-server
|
||||||
pullPolicy: IfNotPresent
|
pullPolicy: IfNotPresent
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 1280Mi
|
|
||||||
# Node trusts its built-in roots, which still include the
|
# Node trusts its built-in roots, which still include the
|
||||||
# ones above the compromised eMudhra intermediates. Use
|
# ones above the compromised eMudhra intermediates. Use
|
||||||
# patched-ca-bundle through OpenSSL until Node ships a fix.
|
# patched-ca-bundle through OpenSSL until Node ships a fix.
|
||||||
@@ -117,19 +109,10 @@ spec:
|
|||||||
image:
|
image:
|
||||||
repository: ghcr.io/immich-app/immich-machine-learning
|
repository: ghcr.io/immich-app/immich-machine-learning
|
||||||
pullPolicy: IfNotPresent
|
pullPolicy: IfNotPresent
|
||||||
# Idle at night (234Mi); the models take over a GiB when
|
|
||||||
# it works.
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 50m
|
|
||||||
memory: 1Gi
|
|
||||||
env:
|
env:
|
||||||
TRANSFORMERS_CACHE: /cache
|
TRANSFORMERS_CACHE: /cache
|
||||||
HF_XET_CACHE: /cache/huggingface-xet
|
HF_XET_CACHE: /cache/huggingface-xet
|
||||||
MPLCONFIGDIR: /cache/matplotlib-config
|
MPLCONFIGDIR: /cache/matplotlib-config
|
||||||
# The optimized models of v3.3.0: faster, less memory,
|
|
||||||
# and the same results, so no jobs to run again.
|
|
||||||
MACHINE_LEARNING_MODEL_REVISION: v2
|
|
||||||
# Trust store without the compromised eMudhra
|
# Trust store without the compromised eMudhra
|
||||||
# intermediates; requests would otherwise use certifi.
|
# intermediates; requests would otherwise use certifi.
|
||||||
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
||||||
@@ -142,19 +125,11 @@ spec:
|
|||||||
globalMounts:
|
globalMounts:
|
||||||
- path: /etc/ssl/certs
|
- path: /etc/ssl/certs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
# The models and their one-off preparation, kept across restarts.
|
|
||||||
# Downloaded again if lost, so no backups: the no-backup group
|
|
||||||
# keeps it out of Longhorn's default one (backup-daily), and
|
|
||||||
# Velero skips that label too (velero/skip-emptydir-policy.yaml).
|
|
||||||
cache:
|
cache:
|
||||||
enabled: true
|
enabled: true
|
||||||
type: persistentVolumeClaim
|
# Optional: to avoid downloading the ML models every start, use
|
||||||
storageClass: longhorn
|
# a persistentVolumeClaim with size, accessMode and storageClass
|
||||||
accessMode: ReadWriteOnce
|
type: emptyDir
|
||||||
size: 10Gi
|
|
||||||
labels:
|
|
||||||
recurring-job.longhorn.io/source: enabled
|
|
||||||
recurring-job-group.longhorn.io/no-backup: enabled
|
|
||||||
|
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
|
|||||||
@@ -9,10 +9,6 @@ spec:
|
|||||||
# CNPG image with VectorChord and pgvector, which immich 2.x and 3.x use
|
# CNPG image with VectorChord and pgvector, which immich 2.x and 3.x use
|
||||||
# for its vector indexes (pgvecto.rs is gone in immich 3.0).
|
# for its vector indexes (pgvecto.rs is gone in immich 3.0).
|
||||||
imageName: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
|
imageName: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 20m
|
|
||||||
memory: 320Mi
|
|
||||||
|
|
||||||
postgresql:
|
postgresql:
|
||||||
shared_preload_libraries:
|
shared_preload_libraries:
|
||||||
|
|||||||
@@ -16,26 +16,14 @@ spec:
|
|||||||
targetRevision: 25.10.4
|
targetRevision: 25.10.4
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 48Mi
|
|
||||||
metrics:
|
metrics:
|
||||||
enabled: true
|
enabled: true
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: false
|
enabled: false
|
||||||
webhook:
|
webhook:
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 32Mi
|
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: false
|
enabled: false
|
||||||
certController:
|
certController:
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 32Mi
|
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: false
|
enabled: false
|
||||||
syncPolicy:
|
syncPolicy:
|
||||||
|
|||||||
@@ -1,35 +0,0 @@
|
|||||||
# Garage on the NAS
|
|
||||||
|
|
||||||
S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
|
|
||||||
the cluster. Replaced MinIO, which is no longer maintained. One node, data
|
|
||||||
in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
|
|
||||||
3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
|
|
||||||
|
|
||||||
The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
|
|
||||||
in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
|
|
||||||
its PATH.
|
|
||||||
|
|
||||||
## Setup
|
|
||||||
|
|
||||||
`garage.toml` here has placeholders for the two secrets; on the NAS they
|
|
||||||
were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
|
|
||||||
`openssl rand -base64 32` (`admin_token`), and only exist there.
|
|
||||||
|
|
||||||
```
|
|
||||||
cd /volume1/docker/garage
|
|
||||||
docker compose up -d
|
|
||||||
g="docker exec garage /garage"
|
|
||||||
$g layout assign -z nas -c 800G <node id from $g status>
|
|
||||||
$g layout apply --version 1
|
|
||||||
$g bucket create velero-backups
|
|
||||||
$g key create velero
|
|
||||||
$g bucket allow --read --write --key velero velero-backups
|
|
||||||
```
|
|
||||||
|
|
||||||
Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
|
|
||||||
`secret_access_key`) and reaches Velero through
|
|
||||||
`velero/external-secret.yaml`. Without the cluster, `$g key info velero
|
|
||||||
--show-secret` on the NAS shows it again.
|
|
||||||
|
|
||||||
Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
|
|
||||||
after merging, `docker compose pull && docker compose up -d` on the NAS.
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
services:
|
|
||||||
garage:
|
|
||||||
image: dxflrs/garage:v2.5.0
|
|
||||||
container_name: garage
|
|
||||||
restart: unless-stopped
|
|
||||||
ports:
|
|
||||||
- "3900:3900" # S3 API
|
|
||||||
- "3903:3903" # admin API
|
|
||||||
volumes:
|
|
||||||
- ./garage.toml:/etc/garage.toml:ro
|
|
||||||
- ./meta:/var/lib/garage/meta
|
|
||||||
- ./data:/var/lib/garage/data
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# One node on the Synology. Velero's kopia data is encrypted, so
|
|
||||||
# compression would only cost CPU.
|
|
||||||
#
|
|
||||||
# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
|
|
||||||
# shutdown, and a single node has no other node to repair from. With
|
|
||||||
# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
|
|
||||||
# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
|
|
||||||
# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
|
|
||||||
# with `garage convert-db`.
|
|
||||||
metadata_dir = "/var/lib/garage/meta"
|
|
||||||
data_dir = "/var/lib/garage/data"
|
|
||||||
db_engine = "sqlite"
|
|
||||||
metadata_fsync = true
|
|
||||||
data_fsync = true
|
|
||||||
metadata_auto_snapshot_interval = "6h"
|
|
||||||
replication_factor = 1
|
|
||||||
compression_level = "none"
|
|
||||||
|
|
||||||
rpc_bind_addr = "[::]:3901"
|
|
||||||
rpc_public_addr = "127.0.0.1:3901"
|
|
||||||
rpc_secret = "@RPC_SECRET@"
|
|
||||||
|
|
||||||
[s3_api]
|
|
||||||
s3_region = "garage"
|
|
||||||
api_bind_addr = "[::]:3900"
|
|
||||||
|
|
||||||
[admin]
|
|
||||||
api_bind_addr = "[::]:3903"
|
|
||||||
admin_token = "@ADMIN_TOKEN@"
|
|
||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: ntfy
|
- name: ntfy
|
||||||
image: binwiederhier/ntfy:v2.34.0
|
image: binwiederhier/ntfy:v2.28.0
|
||||||
args: ["serve"]
|
args: ["serve"]
|
||||||
env:
|
env:
|
||||||
- name: NTFY_BASE_URL
|
- name: NTFY_BASE_URL
|
||||||
|
|||||||
+17
-27
@@ -6,7 +6,7 @@
|
|||||||
],
|
],
|
||||||
"argocd": {
|
"argocd": {
|
||||||
"managerFilePatterns": [
|
"managerFilePatterns": [
|
||||||
"/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
|
"/apps/templates/.+\\.yaml$/"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"helm-values": {
|
"helm-values": {
|
||||||
@@ -16,7 +16,12 @@
|
|||||||
},
|
},
|
||||||
"kubernetes": {
|
"kubernetes": {
|
||||||
"managerFilePatterns": [
|
"managerFilePatterns": [
|
||||||
"/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
|
"/apps/templates/.+\\.yaml$/",
|
||||||
|
"/serpbear/.+\\.yaml$/",
|
||||||
|
"/bolletjestrui/.+\\.yaml$/",
|
||||||
|
"/gitea/.+\\.yaml$/",
|
||||||
|
"/gatus/.+\\.yaml$/",
|
||||||
|
"/ntfy/.+\\.yaml$/"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"customManagers": [
|
"customManagers": [
|
||||||
@@ -26,20 +31,19 @@
|
|||||||
"/apps/templates/.+\\.yaml$/"
|
"/apps/templates/.+\\.yaml$/"
|
||||||
],
|
],
|
||||||
"matchStrings": [
|
"matchStrings": [
|
||||||
"repoURL: (?<registryUrl>.*?)\\n\\s+targetRevision: (?<currentValue>[^\\s]+)\\n\\s+chart: (?<depName>[^\\s]+)"
|
"repoURL: https://github.com/(?<depName>[^/]+/[^/\\.]+)\\.git\\n\\s+targetRevision: (?<currentValue>.*)"
|
||||||
],
|
],
|
||||||
"datasourceTemplate": "helm"
|
"datasourceTemplate": "github-tags"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"customType": "regex",
|
"customType": "regex",
|
||||||
"description": "CloudNativePG clusters name their image in imageName",
|
|
||||||
"managerFilePatterns": [
|
"managerFilePatterns": [
|
||||||
"/^[^/]+/postgresql-cluster[^/]*\\.yaml$/"
|
"/apps/templates/.+\\.yaml$/"
|
||||||
],
|
],
|
||||||
"matchStrings": [
|
"matchStrings": [
|
||||||
"imageName: (?<depName>[^:\\s]+):(?<currentValue>\\S+)"
|
"repoURL: (?<registryUrl>.*?)\\n\\s+targetRevision: (?<currentValue>[^\\s]+)\\n\\s+chart: (?<depName>[^\\s]+)"
|
||||||
],
|
],
|
||||||
"datasourceTemplate": "docker"
|
"datasourceTemplate": "helm"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"packageRules": [
|
"packageRules": [
|
||||||
@@ -52,26 +56,12 @@
|
|||||||
"separateMinorPatch": true
|
"separateMinorPatch": true
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "CI renders with the same Helm as Argo CD (3.5 ships Helm 4.2); move it together with Argo CD",
|
"matchDatasources": [
|
||||||
"matchFileNames": [
|
"github-tags"
|
||||||
".github/workflows/**"
|
|
||||||
],
|
],
|
||||||
"matchDepNames": [
|
"semanticCommits": "enabled",
|
||||||
"helm/helm"
|
"commitMessageTopic": "{{depName}}",
|
||||||
],
|
"separateMinorPatch": true
|
||||||
"enabled": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"description": "A PostgreSQL major upgrade needs pg_upgrade or a dump, not a tag bump",
|
|
||||||
"matchDepNames": [
|
|
||||||
"ghcr.io/cloudnative-pg/postgresql",
|
|
||||||
"ghcr.io/tensorchord/cloudnative-vectorchord",
|
|
||||||
"ghcr.io/tensorchord/cloudnative-pgvecto.rs"
|
|
||||||
],
|
|
||||||
"matchUpdateTypes": [
|
|
||||||
"major"
|
|
||||||
],
|
|
||||||
"enabled": false
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -56,23 +56,6 @@ spec:
|
|||||||
ports:
|
ports:
|
||||||
- containerPort: 3000
|
- containerPort: 3000
|
||||||
protocol: TCP
|
protocol: TCP
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 160Mi
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /login
|
|
||||||
port: 3000
|
|
||||||
periodSeconds: 10
|
|
||||||
timeoutSeconds: 5
|
|
||||||
livenessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 3000
|
|
||||||
initialDelaySeconds: 60
|
|
||||||
periodSeconds: 30
|
|
||||||
timeoutSeconds: 10
|
|
||||||
failureThreshold: 5
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /app/data
|
- mountPath: /app/data
|
||||||
name: data
|
name: data
|
||||||
|
|||||||
+3
-57
@@ -15,13 +15,6 @@ in the app folders. Sources:
|
|||||||
same, with the Application's kustomize patches applied
|
same, with the Application's kustomize patches applied
|
||||||
Every YAML file in the repo is also checked for duplicate keys, which
|
Every YAML file in the repo is also checked for duplicate keys, which
|
||||||
YAML parsers resolve silently by keeping the last one.
|
YAML parsers resolve silently by keeping the last one.
|
||||||
|
|
||||||
Repo rules, on what this repo's own folders render (not on charts or
|
|
||||||
remote bases):
|
|
||||||
- no Secret with data: secrets come from Vault through ESO
|
|
||||||
- every image has a tag other than latest
|
|
||||||
- warnings (annotations on the PR) for containers without requests or
|
|
||||||
probes
|
|
||||||
"""
|
"""
|
||||||
import fnmatch
|
import fnmatch
|
||||||
import os
|
import os
|
||||||
@@ -29,7 +22,6 @@ import pathlib
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
@@ -46,15 +38,7 @@ failures = []
|
|||||||
|
|
||||||
|
|
||||||
def run(cmd, **kw):
|
def run(cmd, **kw):
|
||||||
# Registries rate-limit now and then (ghcr.io answers 429); try again.
|
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
|
||||||
for attempt in range(3):
|
|
||||||
try:
|
|
||||||
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
transient = any(s in (e.stderr or "") for s in ("429", "toomanyrequests", "timeout", "connection reset"))
|
|
||||||
if not transient or attempt == 2:
|
|
||||||
raise
|
|
||||||
time.sleep(10 * (attempt + 1))
|
|
||||||
|
|
||||||
|
|
||||||
class UniqueKeyLoader(yaml.SafeLoader):
|
class UniqueKeyLoader(yaml.SafeLoader):
|
||||||
@@ -94,41 +78,7 @@ def check_duplicate_keys():
|
|||||||
failures.append(f"{path.relative_to(ROOT)}: {e}")
|
failures.append(f"{path.relative_to(ROOT)}: {e}")
|
||||||
|
|
||||||
|
|
||||||
def pod_specs(doc):
|
def validate(label, manifests):
|
||||||
spec = doc.get("spec") or {}
|
|
||||||
kind = doc.get("kind")
|
|
||||||
if kind == "Pod":
|
|
||||||
yield spec
|
|
||||||
elif kind == "CronJob":
|
|
||||||
yield spec.get("jobTemplate", {}).get("spec", {}).get("template", {}).get("spec", {})
|
|
||||||
elif kind in ("Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"):
|
|
||||||
yield spec.get("template", {}).get("spec", {})
|
|
||||||
|
|
||||||
|
|
||||||
def check_rules(label, docs):
|
|
||||||
for d in docs:
|
|
||||||
name = f"{d.get('kind')}/{d.get('metadata', {}).get('name')}"
|
|
||||||
if d.get("kind") == "Secret" and (d.get("data") or d.get("stringData")):
|
|
||||||
failures.append(f"{label}: {name} has data in git; put it in Vault and use an ExternalSecret")
|
|
||||||
for spec in pod_specs(d):
|
|
||||||
containers = spec.get("containers", []) + spec.get("initContainers", [])
|
|
||||||
for c in containers:
|
|
||||||
image = c.get("image", "")
|
|
||||||
tag = image.split("@")[0].rsplit("/", 1)[-1].partition(":")[2]
|
|
||||||
if "@" not in image and tag in ("", "latest"):
|
|
||||||
failures.append(f"{label}: {name} container {c.get('name')} has no fixed tag ({image})")
|
|
||||||
for c in spec.get("containers", []):
|
|
||||||
missing = []
|
|
||||||
if not c.get("resources", {}).get("requests"):
|
|
||||||
missing.append("requests")
|
|
||||||
if d.get("kind") in ("Deployment", "StatefulSet", "DaemonSet") and not (
|
|
||||||
c.get("readinessProbe") or c.get("livenessProbe")):
|
|
||||||
missing.append("probes")
|
|
||||||
if missing:
|
|
||||||
print(f"::warning title={label}::{name} container {c.get('name')} has no {' or '.join(missing)}")
|
|
||||||
|
|
||||||
|
|
||||||
def validate(label, manifests, rules=False):
|
|
||||||
"""kubeconform on rendered manifests; returns the parsed documents."""
|
"""kubeconform on rendered manifests; returns the parsed documents."""
|
||||||
try:
|
try:
|
||||||
docs = [d for d in yaml.safe_load_all(manifests) if d]
|
docs = [d for d in yaml.safe_load_all(manifests) if d]
|
||||||
@@ -153,8 +103,6 @@ def validate(label, manifests, rules=False):
|
|||||||
for l in lines[:-1]:
|
for l in lines[:-1]:
|
||||||
print(f" {l}")
|
print(f" {l}")
|
||||||
failures.append(f"{label}: {summary or result.stderr.strip()}")
|
failures.append(f"{label}: {summary or result.stderr.strip()}")
|
||||||
if rules and docs:
|
|
||||||
check_rules(label, [d for d in docs if isinstance(d, dict)])
|
|
||||||
return docs or []
|
return docs or []
|
||||||
|
|
||||||
|
|
||||||
@@ -290,9 +238,7 @@ def main():
|
|||||||
failures.append(f"{label}: render failed")
|
failures.append(f"{label}: render failed")
|
||||||
continue
|
continue
|
||||||
if manifests is not None:
|
if manifests is not None:
|
||||||
own = src.get("repoURL") in OWN_REPOS and not src.get("chart") and not (
|
validate(label, manifests)
|
||||||
ROOT / src.get("path", ".") / "Chart.yaml").exists()
|
|
||||||
validate(label, manifests, rules=own)
|
|
||||||
|
|
||||||
if failures:
|
if failures:
|
||||||
print(f"\n{len(failures)} problem(s):")
|
print(f"\n{len(failures)} problem(s):")
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module dqlite-vacuum
|
|||||||
go 1.27.3
|
go 1.27.3
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.4
|
github.com/canonical/go-dqlite/v3 v3.0.1
|
||||||
gopkg.in/yaml.v2 v2.4.0
|
gopkg.in/yaml.v2 v2.4.0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -2,10 +2,7 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
|
|||||||
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
|
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
|
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@@ -29,9 +26,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
|
||||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
|||||||
@@ -3,10 +3,6 @@ global:
|
|||||||
tlsDisable: true
|
tlsDisable: true
|
||||||
injector:
|
injector:
|
||||||
enabled: true
|
enabled: true
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 32Mi
|
|
||||||
# The chart's default anti-affinity forbids two injector pods on one
|
# The chart's default anti-affinity forbids two injector pods on one
|
||||||
# node, so on this single node a rolling update can never start the new
|
# node, so on this single node a rolling update can never start the new
|
||||||
# pod while the old one runs. Recreate removes the old pod first, which
|
# pod while the old one runs. Recreate removes the old pod first, which
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
|
|
||||||
# AWS credentials file format the aws plugin and kopia read.
|
|
||||||
apiVersion: external-secrets.io/v1
|
|
||||||
kind: ExternalSecret
|
|
||||||
metadata:
|
|
||||||
name: velero-s3-credentials
|
|
||||||
spec:
|
|
||||||
secretStoreRef:
|
|
||||||
name: vault-backend
|
|
||||||
kind: ClusterSecretStore
|
|
||||||
refreshInterval: 15m
|
|
||||||
target:
|
|
||||||
name: velero-s3-credentials
|
|
||||||
creationPolicy: Owner
|
|
||||||
template:
|
|
||||||
engineVersion: v2
|
|
||||||
data:
|
|
||||||
cloud: |
|
|
||||||
[default]
|
|
||||||
aws_access_key_id={{ .access_key_id }}
|
|
||||||
aws_secret_access_key={{ .secret_access_key }}
|
|
||||||
data:
|
|
||||||
- secretKey: access_key_id
|
|
||||||
remoteRef:
|
|
||||||
key: kv/velero-s3
|
|
||||||
property: access_key_id
|
|
||||||
- secretKey: secret_access_key
|
|
||||||
remoteRef:
|
|
||||||
key: kv/velero-s3
|
|
||||||
property: secret_access_key
|
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
# Volume policy for the backup schedules. emptyDir volumes hold nothing
|
# Volume policy for the backup schedules. emptyDir volumes hold nothing
|
||||||
# worth keeping, and backing them up fails when the pod is gone before its
|
# worth keeping, and backing them up fails when the pod is gone before its
|
||||||
# volume is read, as happens with Velero's own repository maintenance jobs.
|
# volume is read, as happens with Velero's own repository maintenance jobs.
|
||||||
# PVCs in Longhorn's no-backup group (caches, such as immich's ML models)
|
|
||||||
# are skipped too.
|
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
metadata:
|
metadata:
|
||||||
@@ -16,8 +14,3 @@ data:
|
|||||||
- emptyDir
|
- emptyDir
|
||||||
action:
|
action:
|
||||||
type: skip
|
type: skip
|
||||||
- conditions:
|
|
||||||
pvcLabels:
|
|
||||||
recurring-job-group.longhorn.io/no-backup: enabled
|
|
||||||
action:
|
|
||||||
type: skip
|
|
||||||
@@ -68,10 +68,6 @@ spec:
|
|||||||
nodeAgent:
|
nodeAgent:
|
||||||
containerSecurityContext:
|
containerSecurityContext:
|
||||||
privileged: true
|
privileged: true
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 10m
|
|
||||||
memory: 64Mi
|
|
||||||
# 512Mi got the server OOMKilled 314 times, including halfway
|
# 512Mi got the server OOMKilled 314 times, including halfway
|
||||||
# through a backup of ~14.7k items.
|
# through a backup of ~14.7k items.
|
||||||
resources:
|
resources:
|
||||||
|
|||||||
Reference in new issue
Block a user