mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 22:29:09 +01:00
Compare commits
33
Commits
68b692a112
...
d85362ce3b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d85362ce3b | ||
|
|
018f528fb3 | ||
|
|
714712b862 | ||
|
|
7dc13466cb | ||
|
|
9b77596bcb | ||
|
|
fbf2cb2d92 | ||
|
|
122873ca5c | ||
|
|
71449dd185 | ||
|
|
297d2dffa5 | ||
|
|
46117ee658 | ||
|
|
2a5bbc7f44 | ||
|
|
d7b0372f1a | ||
|
|
0ad6ca816c | ||
|
|
3e252799a4 | ||
|
|
bfd5619e9e | ||
|
|
0b38d8dec8 | ||
|
|
083353fb9d | ||
|
|
23f9c5c60a | ||
|
|
a8bf3d5b64 | ||
|
|
618954ae2c | ||
|
|
44dc85d306 | ||
|
|
b60735f6b0 | ||
|
|
3a57e5be48 | ||
|
|
1611f5d1ef | ||
|
|
5bb1422594 | ||
|
|
72f49bed51 | ||
|
|
6b2f5855d7 | ||
|
|
c25b8883da | ||
|
|
18f9e2efe6 | ||
|
|
5f5f5b01f0 | ||
|
|
6206ab5a74 | ||
|
|
075b3686c3 | ||
|
|
45d1ea0d93 |
No files matched your search
@@ -99,6 +99,10 @@ spec:
|
||||
- name: REQUESTS_CA_BUNDLE
|
||||
value: /etc/ssl/certs/ca-certificates.crt
|
||||
server:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 70m
|
||||
memory: 560Mi
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
@@ -184,6 +188,10 @@ spec:
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
worker:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 336Mi
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://charts.jetstack.io
|
||||
targetRevision: v1.21.2
|
||||
targetRevision: v1.21.3
|
||||
chart: cert-manager
|
||||
helm:
|
||||
parameters:
|
||||
@@ -34,6 +34,20 @@ spec:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
cainjector:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 80Mi
|
||||
webhook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
||||
@@ -24,6 +24,10 @@ spec:
|
||||
additionalArgs:
|
||||
- --leader-lease-duration=60
|
||||
- --leader-renew-deadline=40
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cnpg-system
|
||||
|
||||
@@ -17,7 +17,23 @@ spec:
|
||||
sources:
|
||||
- chart: external-secrets
|
||||
repoURL: https://charts.external-secrets.io
|
||||
targetRevision: 2.13.0
|
||||
targetRevision: 2.14.0
|
||||
helm:
|
||||
values: |
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
certController:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
webhook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: external-secrets
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://groundhog2k.github.io/helm-charts/
|
||||
targetRevision: 0.15.0
|
||||
targetRevision: 0.16.7
|
||||
chart: wordpress
|
||||
helm:
|
||||
values: |
|
||||
@@ -101,6 +101,23 @@ spec:
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
|
||||
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
|
||||
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
|
||||
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
|
||||
# crawler asking for 88 pages at once got the container OOMKilled on
|
||||
# 2026-10-25. With 12 workers a burst waits in the listen backlog
|
||||
# instead. Listen is the chart's default (containerPort).
|
||||
apachePortsConfig: |
|
||||
Listen 8000
|
||||
<IfModule mpm_prefork_module>
|
||||
StartServers 4
|
||||
MinSpareServers 2
|
||||
MaxSpareServers 6
|
||||
ServerLimit 12
|
||||
MaxRequestWorkers 12
|
||||
MaxConnectionsPerChild 1000
|
||||
</IfModule>
|
||||
|
||||
# Custom PHP configuration
|
||||
customPhpConfig: |
|
||||
upload_max_filesize = 64M
|
||||
|
||||
@@ -7,7 +7,7 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
||||
targetRevision: v8.2.1
|
||||
targetRevision: v8.7.0
|
||||
path: deploy/kubernetes/snapshot-controller
|
||||
kustomize:
|
||||
patches:
|
||||
|
||||
@@ -21,4 +21,5 @@ patches:
|
||||
- path: overlays/argocd-cm.patch.yaml
|
||||
- path: overlays/argocd-rbac-cm.patch.yaml
|
||||
- path: overlays/argocd-cmd-params-cm.patch.yaml
|
||||
- path: overlays/patched-ca-bundle.patch.yaml
|
||||
- path: overlays/patched-ca-bundle.patch.yaml
|
||||
- path: overlays/resources.patch.yaml
|
||||
@@ -0,0 +1,100 @@
|
||||
# Requests from what the components used on 2026-10-25 (kubectl top every
|
||||
# 5 minutes; highest value plus 10%). No limits until there are daytime
|
||||
# numbers, see docs/roadmap.md.
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: argocd-application-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-application-controller
|
||||
resources:
|
||||
requests:
|
||||
cpu: 150m
|
||||
memory: 704Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-repo-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-repo-server
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-server
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-redis
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 96Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-applicationset-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-applicationset-controller
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-notifications-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-notifications-controller
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-dex-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: dex
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
@@ -8,6 +8,10 @@ spec:
|
||||
# start an in-place major upgrade. CNPG filled this in at creation.
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
|
||||
primaryUpdateStrategy: unsupervised
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: authentik
|
||||
|
||||
@@ -33,8 +33,6 @@
|
||||
set -euo pipefail
|
||||
|
||||
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
|
||||
MINIO=${MINIO:-http://192.168.1.99:9000}
|
||||
BUCKET=velero-backups
|
||||
PORT=${PORT:-16443}
|
||||
K3S=rancher/k3s:v1.34.9-k3s1
|
||||
RCLONE=rclone/rclone:1.75.1
|
||||
@@ -73,6 +71,18 @@ wait_for() {
|
||||
done
|
||||
}
|
||||
|
||||
# Endpoint, region and bucket as Velero itself uses them. MinIO answers a
|
||||
# request signed for the wrong region (rclone defaults to us-east-1) with a
|
||||
# bare 400 on HEAD.
|
||||
if [[ -n ${FROM_VOLUME:-} ]]; then
|
||||
BUCKET=${BUCKET:-velero-backups}
|
||||
else
|
||||
bsl=$(pk -n velero get backupstoragelocations default -o json)
|
||||
MINIO=${MINIO:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
|
||||
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
|
||||
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
|
||||
fi
|
||||
|
||||
backup=${1:-}
|
||||
shift || true
|
||||
namespaces=("$@")
|
||||
@@ -94,7 +104,7 @@ echo "== copy from the NAS (read-only)"
|
||||
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
||||
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
||||
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
||||
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env")
|
||||
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$MINIO" "$REGION" >> "$WORK/nas.env")
|
||||
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
||||
copy "backups/$backup"
|
||||
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
||||
|
||||
@@ -123,15 +123,13 @@ D. auth.scoutingmbg.nl (2026-10-25)
|
||||
|
||||
## Open points
|
||||
|
||||
- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
|
||||
foto.scoutingmbg.nl from a CNAME to `mbg.hensen.io` (the home IP) to a
|
||||
CNAME to `rubenhensen.nl`, which is node001 on Hetzner. Only ns2 has it
|
||||
(serial 2026102471); ns1 is stuck on serial 2026060640 from June and
|
||||
still answers `mbg.hensen.io`. node001 now tunnels foto.scoutingmbg.nl to
|
||||
the home cluster like its other hosts, so both answers work. Keep
|
||||
hensen.io until ns1 has caught up; ask the DNS host to fix ns1, and
|
||||
decide whether foto should stay behind node001 or point straight home
|
||||
(CNAME `scoutingmbg.nl`)
|
||||
- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
|
||||
scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
|
||||
all the records it moved, and node001 tunnels all four home. ns1 lagged
|
||||
behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
|
||||
uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
|
||||
`a` now means node001, which is fine: the WordPress site sends no mail
|
||||
and the group's mail goes through Google (`mx`)
|
||||
- Public shared links become public for real once sol-auth is gone.
|
||||
- Photos leiding back up from their phones stay private until they add
|
||||
them to a shared album.
|
||||
@@ -38,14 +38,15 @@ between them live.
|
||||
|
||||
## Before: shrink what has to move
|
||||
|
||||
- [ ] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
||||
- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
||||
snapshots from before October 2026, nearly all on immich-claim
|
||||
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
|
||||
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
|
||||
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
||||
don't depend on them. Deleting a snapshot merges it into the next one,
|
||||
which writes a lot, so do this in a quiet hour, one volume at a time.
|
||||
Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
|
||||
Happened on 2026-10-24, from a Claude Code session at Ruben's request
|
||||
(the 45 snapshots from before October 2026, small volumes first): immich-claim's
|
||||
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
||||
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
||||
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
||||
|
||||
+74
-40
@@ -50,8 +50,9 @@ group's ns2: ns1 is stuck on an old serial and still answers
|
||||
mbg.hensen.io (docs/immich-scouts-online.md).
|
||||
|
||||
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
||||
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
|
||||
rubenhensen.nl too
|
||||
- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
|
||||
`rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
|
||||
cluster uses hensen.io anymore. hensen.io can expire
|
||||
|
||||
### 2. Off-site backups
|
||||
|
||||
@@ -99,8 +100,17 @@ friend.
|
||||
ReadOnly location: data restored byte for byte (same sha256), deployment
|
||||
available, everything removed afterwards. Two pitfalls found on the way
|
||||
are in the script's header. Not yet run against the real backups:
|
||||
- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from
|
||||
the cluster), then schedule it
|
||||
- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
|
||||
daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
|
||||
volume came back with its backed-up byte count (actual-budget 8.4 MB,
|
||||
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
|
||||
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
|
||||
MinIO runs region `minio`; the script now reads endpoint, region and
|
||||
bucket from Velero's location. Copying freshrss's kopia repository took
|
||||
14 of the ~17 minutes
|
||||
- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
|
||||
the cluster itself: the Mac (launchd) or node001, ideally with a
|
||||
read-only MinIO user
|
||||
|
||||
### 3. authentik 2026.2.1 → 2026.8.5
|
||||
|
||||
@@ -155,8 +165,8 @@ unseals it.
|
||||
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
||||
doesn't exist, so the OIDC mount's listing visibility was never set.
|
||||
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
||||
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
|
||||
places outside this cluster; without it the data in Vault is lost
|
||||
- [x] There is one unseal key (1 share, threshold 1), kept safe outside
|
||||
this cluster (2026-10-25); without it the data in Vault is lost
|
||||
- [x] Unsealing is in the SSD install steps
|
||||
- [ ] After a week without problems: delete the three snapshots
|
||||
|
||||
@@ -267,14 +277,42 @@ Counted 2026-10-25 over the 73 running containers:
|
||||
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
||||
default) still have none; `default` can reach the LAN and everything
|
||||
without a policy
|
||||
- [ ] Requests: PR #33 (2026-10-25, CI green, not merged) sets them for
|
||||
everything that had none except the Vault server, from `kubectl top`
|
||||
every 5 minutes overnight (highest memory plus 10%, CPU at p95, 10m
|
||||
floor); about 1.2 CPU and 7 GiB together. Merging restarts most pods,
|
||||
including both CNPG instances (half a minute without a database for
|
||||
authentik and immich)
|
||||
- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
|
||||
had none except the Vault server, from `kubectl top` every 5 minutes
|
||||
from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
|
||||
floor). Rolled out in about eight minutes; both CNPG instances
|
||||
restarted and came back healthy, the OIDC endpoints, the logins and
|
||||
immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
|
||||
1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
|
||||
requested. Still without requests: Longhorn (22 containers, its chart),
|
||||
the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
|
||||
kube-system
|
||||
- [ ] Memory limits, once there are daytime numbers: the night ones miss
|
||||
immich's machine learning at work (it idles at 234Mi)
|
||||
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
|
||||
2026-10-25, its third restart: a crawler asked for 88 pages and assets
|
||||
within a few seconds (normally ~6 a minute), Apache started a PHP worker
|
||||
for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
|
||||
the container went past its 1Gi limit. Unrelated to the
|
||||
NetworkPolicies. Now capped at 12 workers through the chart's
|
||||
`apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
|
||||
with `apache2ctl -t` in the same image first. Replaying the burst (90
|
||||
page requests, 40 at a time): all 200, at most 13 Apache processes and
|
||||
572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
|
||||
peak) instead of a crash
|
||||
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
|
||||
cluster between Envoy and WordPress that keeps anonymous GETs for a few
|
||||
minutes and passes anything with a WordPress cookie through. A page
|
||||
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
|
||||
waiting in a burst is queueing; the cap alone already prevents the crash
|
||||
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
|
||||
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
|
||||
little here, `wordpress-importer` is usually for a one-off import, and
|
||||
`code-snippets` runs PHP from the dashboard
|
||||
- [ ] Rate limiting per visitor in Envoy needs the client's address, which
|
||||
node001 loses: it passes TLS through, so every request arrives from
|
||||
46.224.26.65. That needs the PROXY protocol from node001's nginx to Envoy
|
||||
(ClientTrafficPolicy `enableProxyProtocol`)
|
||||
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
|
||||
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
|
||||
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
|
||||
@@ -322,34 +360,24 @@ PR before Argo CD applies it.
|
||||
cover every app folder, a regex manager reads `imageName` in
|
||||
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
|
||||
(those need pg_upgrade or a dump). Expect a batch of new PRs
|
||||
(2026-10-25; `renovate-config-validator` passes)
|
||||
- [ ] The open PRs, all validated on 2026-10-25 against main as it is now
|
||||
(rebases requested, so the checks show on the PRs once Renovate has
|
||||
pushed). Merging is yours; merging deploys:
|
||||
- #28 cert-manager v1.21.3: security release, "All users should
|
||||
upgrade" (drops the compromised eMudhra root, fixes an ARI renewal
|
||||
loop and a private key copy path). Merge first
|
||||
- #24 external-snapshotter v8.2.1 → v8.7.0: 8.6 moved VolumeGroupSnapshot
|
||||
to v1 and 8.7 turns its feature gate on; the cluster already serves the
|
||||
v1 CRDs. Merge; close #15
|
||||
- #25 WordPress chart 0.15.0 → 0.16.7: the image is pinned
|
||||
(7.1.3-php8.3-apache), so the render only changes labels and
|
||||
checksums (a pod restart). Merge; #19 (0.15.1) is superseded
|
||||
- #26 Gitea 1.27.3 → 28.2.0 (Gitea renumbered after 1.27): security
|
||||
fixes; the breaking changes are to egress host lists
|
||||
(`ALLOWED_DOMAINS` and friends), none of which are set here. Database
|
||||
migrations can't be undone, so merge after a fresh backup
|
||||
- #23 ntfy v2.28.0 → v2.34.0: six releases in 17 days; 2.31.0 had a
|
||||
database load bug that ntfy.sh rolled back (fixed in 2.31.1). No
|
||||
breaking migrations. Fine, or wait a few days for 2.34 to settle
|
||||
- #17 external-secrets 2.13.0 → 2.14.0: no breaking changes in the
|
||||
notes; every secret goes through it, so merge when there's time to
|
||||
watch a force-sync
|
||||
- #27 go-dqlite v3.0.4 for `tools/dqlite-vacuum`: builds; harmless
|
||||
- #29 yaml.v3 for the same tool: builds, but the code imports yaml.v2,
|
||||
so it only adds an unused requirement. Close
|
||||
- #31 python 3.15.1 for `gatus/backup-check.yaml`: the check's tests
|
||||
(made-up data for every failure) pass on that image. Merge
|
||||
(2026-10-25; `renovate-config-validator` passes). They only come once
|
||||
fewer than 10 Renovate PRs are open (`config:recommended`'s
|
||||
`prConcurrentLimit`); until then they wait under "Rate-Limited" on the
|
||||
Dependency Dashboard (#2)
|
||||
- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
|
||||
v1.21.3 (security release; all three pods on v1.21.3), #24
|
||||
external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
|
||||
v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
|
||||
0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
|
||||
ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
|
||||
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
|
||||
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
|
||||
its own beforehand, the nightly ones are the way back)
|
||||
- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
|
||||
manual rebase (#33 had added values on the next line, which git counts
|
||||
as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
|
||||
ExternalSecrets refreshed on a force-sync; the 25th is the generated
|
||||
`gatus/backup-check-token`, which never refreshes by design
|
||||
|
||||
### 9. One node
|
||||
|
||||
@@ -365,6 +393,12 @@ step after that, if downtime starts to matter.
|
||||
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
||||
completed kopia maintenance jobs of repositories that no longer exist
|
||||
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
||||
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
|
||||
volumes over the file paths `data/config.custom.php` and
|
||||
`data/config-user.custom.php`, so FreshRSS sees two empty directories
|
||||
there and the volumes hold nothing (0 bytes in every backup). Drop both
|
||||
mounts and the two PVCs, or make them ConfigMap files if custom config
|
||||
is wanted
|
||||
- [ ] `kubectl delete namespace secrets`, empty since its app went
|
||||
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
|
||||
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
|
||||
|
||||
@@ -282,6 +282,13 @@ whatever time of day the hourly schedule had drifted to.
|
||||
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
||||
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
||||
their leases at 01:31
|
||||
Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
|
||||
11:30, leaving out the CNPG range cancels): all 67 minutes with errors
|
||||
had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
|
||||
"full" pressure at 45-78%, against a median minute peak of 24 ms. The
|
||||
worst minute after the merges was the rollout of eight merged PRs at
|
||||
2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
|
||||
CNPG and Envoy Gateway held their 60s leases through all of it
|
||||
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
||||
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
||||
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
||||
|
||||
@@ -79,6 +79,10 @@ spec:
|
||||
ports:
|
||||
- containerPort: 80
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 176Mi
|
||||
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
|
||||
# browser.
|
||||
readinessProbe:
|
||||
|
||||
@@ -256,7 +256,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: check
|
||||
image: python:3.13.16-alpine3.24
|
||||
image: python:3.15.1-alpine3.24
|
||||
command: ["python", "/script/check.py"]
|
||||
env:
|
||||
- name: PYTHONDONTWRITEBYTECODE
|
||||
|
||||
@@ -20,7 +20,7 @@ spec:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: gitea
|
||||
image: gitea/gitea:1.27.3
|
||||
image: gitea/gitea:28.2.0
|
||||
env:
|
||||
- name: USER_UID
|
||||
value: "1000"
|
||||
@@ -50,6 +50,10 @@ spec:
|
||||
- containerPort: 3000
|
||||
name: http
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 336Mi
|
||||
# /api/healthz checks the database and the cache.
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
|
||||
@@ -63,6 +63,10 @@ spec:
|
||||
main:
|
||||
containers:
|
||||
main:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
image:
|
||||
repository: docker.io/valkey/valkey
|
||||
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
||||
@@ -85,6 +89,10 @@ spec:
|
||||
image:
|
||||
repository: ghcr.io/immich-app/immich-server
|
||||
pullPolicy: IfNotPresent
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 1280Mi
|
||||
# Node trusts its built-in roots, which still include the
|
||||
# ones above the compromised eMudhra intermediates. Use
|
||||
# patched-ca-bundle through OpenSSL until Node ships a fix.
|
||||
@@ -109,6 +117,12 @@ spec:
|
||||
image:
|
||||
repository: ghcr.io/immich-app/immich-machine-learning
|
||||
pullPolicy: IfNotPresent
|
||||
# Idle at night (234Mi); the models take over a GiB when
|
||||
# it works.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 1Gi
|
||||
env:
|
||||
TRANSFORMERS_CACHE: /cache
|
||||
HF_XET_CACHE: /cache/huggingface-xet
|
||||
|
||||
@@ -9,6 +9,10 @@ spec:
|
||||
# CNPG image with VectorChord and pgvector, which immich 2.x and 3.x use
|
||||
# for its vector indexes (pgvecto.rs is gone in immich 3.0).
|
||||
imageName: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 320Mi
|
||||
|
||||
postgresql:
|
||||
shared_preload_libraries:
|
||||
|
||||
@@ -16,14 +16,26 @@ spec:
|
||||
targetRevision: 25.10.4
|
||||
helm:
|
||||
values: |
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
metrics:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
webhook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
certController:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
syncPolicy:
|
||||
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: ntfy
|
||||
image: binwiederhier/ntfy:v2.28.0
|
||||
image: binwiederhier/ntfy:v2.34.0
|
||||
args: ["serve"]
|
||||
env:
|
||||
- name: NTFY_BASE_URL
|
||||
|
||||
@@ -56,6 +56,10 @@ spec:
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 160Mi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /login
|
||||
|
||||
@@ -3,7 +3,7 @@ module dqlite-vacuum
|
||||
go 1.27.3
|
||||
|
||||
require (
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1
|
||||
github.com/canonical/go-dqlite/v3 v3.0.4
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
|
||||
@@ -2,7 +2,10 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
|
||||
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -26,7 +29,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
|
||||
@@ -3,6 +3,10 @@ global:
|
||||
tlsDisable: true
|
||||
injector:
|
||||
enabled: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
# The chart's default anti-affinity forbids two injector pods on one
|
||||
# node, so on this single node a rolling update can never start the new
|
||||
# pod while the old one runs. Recreate removes the old pod first, which
|
||||
|
||||
@@ -68,6 +68,10 @@ spec:
|
||||
nodeAgent:
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
# 512Mi got the server OOMKilled 314 times, including halfway
|
||||
# through a backup of ~14.7k items.
|
||||
resources:
|
||||
|
||||
Reference in new issue
Block a user