mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 22:29:09 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8708eadc9 | ||
|
|
b29616990c | ||
|
|
9f946d5e68 | ||
|
|
f08737c221 | ||
|
|
65858d3f0b | ||
|
|
b6be49b701 | ||
|
|
d6779a399f | ||
|
|
e042fdd8de | ||
|
|
e8145e12d7 | ||
|
|
3d1b7dd230 | ||
|
|
aaae182844 | ||
|
|
e0e64e1097 | ||
|
|
b4e992d092 | ||
|
|
6ee7fd6f5e | ||
|
|
7db73a100e | ||
|
|
40cd825602 | ||
|
|
bbb56e0e25 | ||
|
|
a4b09fc51e | ||
|
|
0589bb3e08 |
No files matched your search
@@ -12,7 +12,7 @@ spec:
|
|||||||
repoURL: docker.io/envoyproxy
|
repoURL: docker.io/envoyproxy
|
||||||
chart: gateway-helm
|
chart: gateway-helm
|
||||||
# v1.9 supports Kubernetes 1.33 to 1.36
|
# v1.9 supports Kubernetes 1.33 to 1.36
|
||||||
targetRevision: 1.9.3
|
targetRevision: 1.10.0
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
config:
|
config:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://groundhog2k.github.io/helm-charts/
|
repoURL: https://groundhog2k.github.io/helm-charts/
|
||||||
targetRevision: 0.16.7
|
targetRevision: 0.16.8
|
||||||
chart: wordpress
|
chart: wordpress
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ records:
|
|||||||
- name: "ldap"
|
- name: "ldap"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
content: "62.41.86.27"
|
content: "62.41.86.53"
|
||||||
- name: "argocd"
|
- name: "argocd"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
@@ -60,7 +60,7 @@ records:
|
|||||||
- name: "s2"
|
- name: "s2"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
content: "62.41.86.27"
|
content: "62.41.86.53"
|
||||||
# ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
|
# ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
|
||||||
- name: "ntfy1"
|
- name: "ntfy1"
|
||||||
expire: 300
|
expire: 300
|
||||||
@@ -69,7 +69,7 @@ records:
|
|||||||
- name: "ntfy2"
|
- name: "ntfy2"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
content: "62.41.86.27"
|
content: "62.41.86.53"
|
||||||
- name: "@"
|
- name: "@"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: MX
|
type: MX
|
||||||
|
|||||||
+47
-20
@@ -1,22 +1,27 @@
|
|||||||
# Roadmap
|
# Roadmap
|
||||||
|
|
||||||
Where the cluster stands and what is left, as of 2026-10-25. Takes over
|
Where the cluster stands and what is left, as of 2026-10-28. Takes over
|
||||||
from wishlist.txt (January 2026, removed); its items are sorted in at the
|
from wishlist.txt (January 2026, removed); its items are sorted in at the
|
||||||
bottom.
|
bottom.
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
||||||
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
SATA SSD that is 54% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
||||||
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
39 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
||||||
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
|
volume has one replica. authentik 2026.8.5, Vault 2.2.1, immich 3.3.5 on
|
||||||
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to Garage
|
PostgreSQL 17, MariaDB 11.4.13. Memory requests and limits on everything
|
||||||
(MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90 days).
|
but Longhorn, the Vault servers, calico and the LDAP outpost. Backups go
|
||||||
|
to the NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to
|
||||||
|
Garage (MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90
|
||||||
|
days), and CNPG to Garage (a base backup at 03:00, the WAL every 5
|
||||||
|
minutes, 30 days).
|
||||||
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
||||||
through ntfy; s2 also gets an hourly report from inside the cluster on the
|
through ntfy; s2 also gets an hourly report from inside the cluster on the
|
||||||
backups, Vault's seal, the Argo CD apps and the CNPG clusters.
|
backups, Vault's seal, the Argo CD apps and the CNPG clusters.
|
||||||
Every app namespace has NetworkPolicies, and CI renders and validates all
|
Every namespace with pods but kube-system has NetworkPolicies, and CI
|
||||||
apps on every PR. node001 runs mail and tunnels the public hosts home.
|
renders and validates all apps on every PR. node001 runs mail and tunnels
|
||||||
|
the public hosts home.
|
||||||
|
|
||||||
## Follow-ups from this week
|
## Follow-ups from this week
|
||||||
|
|
||||||
@@ -34,7 +39,14 @@ docs/longhorn-second-ssd.md:
|
|||||||
`immich-pg17-pre-v3` only ever was a snapshot
|
`immich-pg17-pre-v3` only ever was a snapshot
|
||||||
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
||||||
(`backend-*.tar`, `dqlite-pre-*`)
|
(`backend-*.tar`, `dqlite-pre-*`)
|
||||||
- [ ] Log in to immich as a jeugdlid (quota 0)
|
- [ ] Log in to immich as a jeugdlid (quota 0): no uploads, albums as
|
||||||
|
viewer, people searchable but not renamable
|
||||||
|
- [x] Searching by face: people are per user in immich, and a partner's
|
||||||
|
library doesn't bring its people along, so since the personal accounts
|
||||||
|
only the group account could. share-sync now puts everyone in the
|
||||||
|
group account's cluster group and shares its people (immich 3.3's
|
||||||
|
person sharing; PR #59, 2026-10-28): 4430 people, 169 named, for each
|
||||||
|
of the 6 accounts, with write access for those who may upload
|
||||||
- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
|
- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
|
||||||
2026-10-25). The snap is held, so microk8s gets no updates at all until
|
2026-10-25). The snap is held, so microk8s gets no updates at all until
|
||||||
someone refreshes it
|
someone refreshes it
|
||||||
@@ -216,7 +228,8 @@ Findings from 2026-10-25:
|
|||||||
22:48, since the old pod waited for its connections to close. The first
|
22:48, since the old pod waited for its connections to close. The first
|
||||||
base backups: 46 MB in 29 s and 776 MB in 3.5 minutes. immich's
|
base backups: 46 MB in 29 s and 776 MB in 3.5 minutes. immich's
|
||||||
`immediate` backup ran before the plugin was up and failed; a manual one
|
`immediate` backup ran before the plugin was up and failed; a manual one
|
||||||
(`immich-postgres-17-first`) went through
|
(`immich-postgres-17-first`) went through. The 03:00 base backups of
|
||||||
|
10-27 and 10-28 completed for both; the failed backup is deleted
|
||||||
- [ ] Restore one of them into a new Cluster (`bootstrap.recovery` from
|
- [ ] Restore one of them into a new Cluster (`bootstrap.recovery` from
|
||||||
the ObjectStore, to a point in time) to see that it works. After that,
|
the ObjectStore, to a point in time) to see that it works. After that,
|
||||||
decide whether Velero still needs to copy the CNPG data directories
|
decide whether Velero still needs to copy the CNPG data directories
|
||||||
@@ -242,11 +255,13 @@ Findings from 2026-10-25:
|
|||||||
MinIO still has the kopia repositories of namespaces that are gone
|
MinIO still has the kopia repositories of namespaces that are gone
|
||||||
(~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
|
(~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
|
||||||
mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
|
mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
|
||||||
whether any of them is worth keeping
|
whether any of them is worth keeping. The dailies of 10-27 and 10-28
|
||||||
|
on Garage completed without errors
|
||||||
- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
|
- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
|
||||||
2026-10-26 next to a 270 MB database, and going by the folders in
|
2026-10-26 next to a 270 MB database, and going by the folders in
|
||||||
`meta/snapshots`, each 6-hourly metadata snapshot takes about 70
|
`meta/snapshots`, each 6-hourly metadata snapshot takes about 70
|
||||||
minutes. Not growing; check again after a week of normal use
|
minutes. Not growing; check again after a week of normal use. 1.8 GB
|
||||||
|
on 2026-10-28
|
||||||
|
|
||||||
### 6. Monitoring
|
### 6. Monitoring
|
||||||
|
|
||||||
@@ -315,13 +330,14 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
OIDC logins of Vault and FreshRSS reach authentik, all 24
|
OIDC logins of Vault and FreshRSS reach authentik, all 24
|
||||||
ExternalSecrets refresh, raft intact; Gitea no longer reaches the NAS
|
ExternalSecrets refresh, raft intact; Gitea no longer reaches the NAS
|
||||||
or Vault, and a pod in `default` no longer reaches Vault or Gitea
|
or Vault, and a pod in `default` no longer reaches Vault or Gitea
|
||||||
- [ ] Certificates couldn't renew in those 11 namespaces: Envoy may only
|
- [x] Certificates couldn't renew in those 11 namespaces: Envoy may only
|
||||||
reach each app's own port, and cert-manager's HTTP-01 solver listens on
|
reach each app's own port, and cert-manager's HTTP-01 solver listens on
|
||||||
8089. ynab's renewal hung from 2026-10-26 17:35 UTC; a policy for the
|
8089. ynab's renewal hung from 2026-10-26 17:35 UTC; a policy for the
|
||||||
solver pods, applied by hand in actual-budget that evening, let it
|
solver pods, applied by hand in actual-budget that evening, let it
|
||||||
through within 20 seconds (now valid until 2027-01-24). PR #51 adds it
|
through within 20 seconds (now valid until 2027-01-24). PR #51 adds it
|
||||||
to all 11. Without it the next renewals hang too: gitea on 11-15,
|
to all 11 (merged 2026-10-26 22:27 UTC). Without it the next renewals
|
||||||
bolletjestrui on 11-18, the rest from 12-03
|
would hang too; the first real test is gitea's on 11-15, then
|
||||||
|
bolletjestrui on 11-18 and the rest from 12-03
|
||||||
- [x] The infrastructure namespaces (cert-manager, cnpg-system,
|
- [x] The infrastructure namespaces (cert-manager, cnpg-system,
|
||||||
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
||||||
default), merged 2026-10-26: PR #54 for envoy-gateway-system (22:31
|
default), merged 2026-10-26: PR #54 for envoy-gateway-system (22:31
|
||||||
@@ -349,6 +365,12 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
sat idle all day (240Mi), so on 2026-10-28 a test image went through
|
sat idle all day (240Mi), so on 2026-10-28 a test image went through
|
||||||
CLIP, face recognition and OCR inside the container: 884Mi with all
|
CLIP, face recognition and OCR inside the container: 884Mi with all
|
||||||
three loaded, limit 2Gi
|
three loaded, limit 2Gi
|
||||||
|
- [ ] authentik's server fails its liveness probe while its database is
|
||||||
|
away (`/-/health/live/` answers 500 when PostgreSQL is down), so each
|
||||||
|
database restart gets the server killed too, as on 2026-10-27 for
|
||||||
|
PostgreSQL 17.11. The login only comes back once the server has
|
||||||
|
started again. A liveness probe that doesn't touch the database would
|
||||||
|
leave it at not ready
|
||||||
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
|
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
|
||||||
2026-10-25, its third restart: a crawler asked for 88 pages and assets
|
2026-10-25, its third restart: a crawler asked for 88 pages and assets
|
||||||
within a few seconds (normally ~6 a minute), Apache started a PHP worker
|
within a few seconds (normally ~6 a minute), Apache started a PHP worker
|
||||||
@@ -414,10 +436,15 @@ every PR.
|
|||||||
fail the run; tested with a planted Secret and an untagged image), plus
|
fail the run; tested with a planted Secret and an untagged image), plus
|
||||||
PR annotations for containers without requests or probes. freshrss's
|
PR annotations for containers without requests or probes. freshrss's
|
||||||
`alpine/git` init container had no tag; now `v2.52.0`
|
`alpine/git` init container had no tag; now `v2.52.0`
|
||||||
- [ ] Automerge patch versions of low-risk apps now that CI runs: PR #52
|
- [x] Automerge patch versions of low-risk apps now that CI runs: PR #52,
|
||||||
(ntfy, gatus, freshrss, serpbear, actual-budget and CI's own tools;
|
merged 2026-10-26 (ntfy, gatus, freshrss, serpbear, actual-budget and
|
||||||
three days after the release, only on weekday evenings and at the
|
CI's own tools; three days after the release, only on weekday evenings
|
||||||
weekend, when the cluster can be reached)
|
and at the weekend, when the cluster can be reached). Nothing has
|
||||||
|
merged itself yet by 2026-10-28
|
||||||
|
- [x] MariaDB stays on 11.4 (PR #60, 2026-10-28): Renovate offered 11.8.9
|
||||||
|
as a minor update (#42, closed), but 11.8 can't be rolled back and has
|
||||||
|
community support until June 2028, against May 2029 for 11.4. Renovate
|
||||||
|
now only offers 11.4 patches; the next LTS step, 12.3, is by hand
|
||||||
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
|
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
|
||||||
`argocd` manager and a custom `github-tags` regex both matched. The
|
`argocd` manager and a custom `github-tags` regex both matched. The
|
||||||
regex is gone; Renovate should close #15 by itself
|
regex is gone; Renovate should close #15 by itself
|
||||||
@@ -510,7 +537,7 @@ critical / high:
|
|||||||
| Image | Crit | High | What to do |
|
| Image | Crit | High | What to do |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; deleted 2026-10-28 |
|
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; deleted 2026-10-28 |
|
||||||
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. Renovate now offers 17.7; a `17.x-standard-trixie` image is more current |
|
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. 17.11 since PR #40 (2026-10-27); not scanned again, a `17.x-standard-trixie` image is more current |
|
||||||
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
||||||
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
||||||
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
||||||
|
|||||||
+2
-2
@@ -62,7 +62,7 @@ node, both PowerDNS Authoritative. PowerDNS LUA records health-check
|
|||||||
addresses and only return the healthy ones:
|
addresses and only return the healthy ones:
|
||||||
|
|
||||||
```
|
```
|
||||||
@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.27'}})"
|
@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.53'}})"
|
||||||
```
|
```
|
||||||
|
|
||||||
The double braces make two groups. PowerDNS returns the first group that has
|
The double braces make two groups. PowerDNS returns the first group that has
|
||||||
@@ -85,7 +85,7 @@ Steps:
|
|||||||
|
|
||||||
1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add
|
1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add
|
||||||
`status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`,
|
`status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`,
|
||||||
plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.27).
|
plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.53).
|
||||||
First check that TransIP accepts NS records for a subdomain.
|
First check that TransIP accepts NS records for a subdomain.
|
||||||
2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and
|
2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and
|
||||||
the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp.
|
the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp.
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: check
|
- name: check
|
||||||
image: python:3.15.1-alpine3.24
|
image: python:3.15.2-alpine3.24
|
||||||
command: ["python", "/script/check.py"]
|
command: ["python", "/script/check.py"]
|
||||||
env:
|
env:
|
||||||
- name: PYTHONDONTWRITEBYTECODE
|
- name: PYTHONDONTWRITEBYTECODE
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: gatus
|
- name: gatus
|
||||||
image: twinproduction/gatus:v5.37.0
|
image: twinproduction/gatus:v5.38.0
|
||||||
env:
|
env:
|
||||||
- name: GATUS_CONFIG_PATH
|
- name: GATUS_CONFIG_PATH
|
||||||
value: /config/config.yaml
|
value: /config/config.yaml
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ spec:
|
|||||||
fsGroup: 1000
|
fsGroup: 1000
|
||||||
containers:
|
containers:
|
||||||
- name: gitea
|
- name: gitea
|
||||||
image: gitea/gitea:28.2.0
|
image: gitea/gitea:28.3.0
|
||||||
env:
|
env:
|
||||||
- name: USER_UID
|
- name: USER_UID
|
||||||
value: "1000"
|
value: "1000"
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
let
|
let
|
||||||
# Public IP of the home network where the k8s cluster ingress lives.
|
# Public IP of the home network where the k8s cluster ingress lives.
|
||||||
# Keep in sync with dns/domains/rubenhensen.nl.yaml.
|
# Keep in sync with dns/domains/rubenhensen.nl.yaml.
|
||||||
homeIP = "62.41.86.27";
|
homeIP = "62.41.86.53";
|
||||||
|
|
||||||
# Hosts tunneled to the home k8s cluster. Each entry gets:
|
# Hosts tunneled to the home k8s cluster. Each entry gets:
|
||||||
# * an SNI map entry for TCP passthrough on :443
|
# * an SNI map entry for TCP passthrough on :443
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: ntfy
|
- name: ntfy
|
||||||
image: binwiederhier/ntfy:v2.34.0
|
image: binwiederhier/ntfy:v2.37.2
|
||||||
args: ["serve"]
|
args: ["serve"]
|
||||||
env:
|
env:
|
||||||
- name: NTFY_BASE_URL
|
- name: NTFY_BASE_URL
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ spec:
|
|||||||
value: --use-openssl-ca
|
value: --use-openssl-ca
|
||||||
- name: SSL_CERT_FILE
|
- name: SSL_CERT_FILE
|
||||||
value: /etc/ssl/certs/ca-certificates.crt
|
value: /etc/ssl/certs/ca-certificates.crt
|
||||||
image: towfiqi/serpbear:3.1.0
|
image: towfiqi/serpbear:3.1.2
|
||||||
name: serpbear
|
name: serpbear
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 3000
|
- containerPort: 3000
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ spec:
|
|||||||
source:
|
source:
|
||||||
chart: velero
|
chart: velero
|
||||||
repoURL: https://vmware-tanzu.github.io/helm-charts
|
repoURL: https://vmware-tanzu.github.io/helm-charts
|
||||||
targetRevision: 12.2.1
|
targetRevision: 12.2.3
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
# Velero builds its repository maintenance jobs from a Deployment
|
# Velero builds its repository maintenance jobs from a Deployment
|
||||||
|
|||||||
Reference in new issue
Block a user