mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-17 06:29:09 +01:00
Compare commits
44
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
829954b883 | ||
|
|
c3830cd9d7 | ||
|
|
6a782ff174 | ||
|
|
6508335914 | ||
|
|
69edd7ef4c | ||
|
|
01470a260c | ||
|
|
f174ddee8f | ||
|
|
62043940e8 | ||
|
|
8a60dbc5ea | ||
|
|
f9111bd17b | ||
|
|
0e6e9dc323 | ||
|
|
288714af21 | ||
|
|
66debf1e3c | ||
|
|
108455de9c | ||
|
|
69444a4ac0 | ||
|
|
30bc62a453 | ||
|
|
2b7b1799a4 | ||
|
|
00de7483d4 | ||
|
|
bde9245497 | ||
|
|
c2bece55d2 | ||
|
|
0bf2245c26 | ||
|
|
af99a17ff4 | ||
|
|
8cc26bb412 | ||
|
|
c6424cc4a5 | ||
|
|
12a1f80902 | ||
|
|
39cc61236f | ||
|
|
42fc278f74 | ||
|
|
94fc9aeb52 | ||
|
|
724afbf4f7 | ||
|
|
cef4466925 | ||
|
|
660cb82922 | ||
|
|
f31a31cfde | ||
|
|
4b7682f12e | ||
|
|
3b5708fd64 | ||
|
|
6cac51ebfa | ||
|
|
c5bb001287 | ||
|
|
7acb7eaa8c | ||
|
|
9e31727298 | ||
|
|
e62728027b | ||
|
|
183686ea34 | ||
|
|
950d4807f2 | ||
|
|
3db1ee84a5 | ||
|
|
04e862d453 | ||
|
|
096eb60f8c |
No files matched your search
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: actualbudget-oidc-client-secret
|
name: actualbudget-oidc-client-secret
|
||||||
|
|||||||
@@ -14,5 +14,8 @@ spec:
|
|||||||
syncPolicy:
|
syncPolicy:
|
||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
|
# From Argo CD 3.3 the ApplicationSet CRD is too big for client-side
|
||||||
|
# apply's last-applied annotation.
|
||||||
|
- ServerSideApply=true
|
||||||
automated:
|
automated:
|
||||||
selfHeal: true
|
selfHeal: true
|
||||||
@@ -9,13 +9,15 @@ spec:
|
|||||||
syncPolicy:
|
syncPolicy:
|
||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
|
# From 0.19 the ESO CRDs are too big for client-side apply.
|
||||||
|
- ServerSideApply=true
|
||||||
automated:
|
automated:
|
||||||
selfHeal: true
|
selfHeal: true
|
||||||
project: default
|
project: default
|
||||||
sources:
|
sources:
|
||||||
- chart: external-secrets
|
- chart: external-secrets
|
||||||
repoURL: https://charts.external-secrets.io
|
repoURL: https://charts.external-secrets.io
|
||||||
targetRevision: 0.10.7
|
targetRevision: 2.13.0
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: external-secrets
|
namespace: external-secrets
|
||||||
@@ -6,7 +6,7 @@ namespace: argocd
|
|||||||
|
|
||||||
# additional resources like ingress rules, cluster and repository secrets.
|
# additional resources like ingress rules, cluster and repository secrets.
|
||||||
resources:
|
resources:
|
||||||
- https://raw.githubusercontent.com/argoproj/argo-cd/refs/tags/v2.14.2/manifests/install.yaml
|
- https://raw.githubusercontent.com/argoproj/argo-cd/refs/tags/v3.4.12/manifests/install.yaml
|
||||||
- ingress.yaml
|
- ingress.yaml
|
||||||
- oidc-secret.yaml
|
- oidc-secret.yaml
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: argocd-oidc-secret
|
name: argocd-oidc-secret
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ metadata:
|
|||||||
app.kubernetes.io/part-of: argocd
|
app.kubernetes.io/part-of: argocd
|
||||||
data:
|
data:
|
||||||
url: https://argocd.rubenhensen.nl
|
url: https://argocd.rubenhensen.nl
|
||||||
|
# Argo CD 3.0 defaults to annotation tracking. Stay on labels through the
|
||||||
|
# upgrade so no app loses track of its resources; switching is a
|
||||||
|
# separate step that needs every app synced once.
|
||||||
|
application.resourceTrackingMethod: label
|
||||||
oidc.config: |
|
oidc.config: |
|
||||||
name: Authentik
|
name: Authentik
|
||||||
issuer: https://authentik.rubenhensen.nl/application/o/argocd/
|
issuer: https://authentik.rubenhensen.nl/application/o/argocd/
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: webhook-es
|
name: webhook-es
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: authentik-secrets
|
name: authentik-secrets
|
||||||
@@ -24,7 +24,7 @@ spec:
|
|||||||
key: kv/authentik
|
key: kv/authentik
|
||||||
property: bootstrap_token
|
property: bootstrap_token
|
||||||
---
|
---
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: authentik-postgres-credentials
|
name: authentik-postgres-credentials
|
||||||
@@ -46,7 +46,7 @@ spec:
|
|||||||
key: kv/authentik
|
key: kv/authentik
|
||||||
property: postgres_password
|
property: postgres_password
|
||||||
---
|
---
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: vault-oidc-client-secret
|
name: vault-oidc-client-secret
|
||||||
@@ -64,7 +64,7 @@ spec:
|
|||||||
key: kv/vault-oidc
|
key: kv/vault-oidc
|
||||||
property: client_secret
|
property: client_secret
|
||||||
---
|
---
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: stalwart-oidc-client-secret
|
name: stalwart-oidc-client-secret
|
||||||
@@ -82,7 +82,7 @@ spec:
|
|||||||
key: kv/stalwart-oidc
|
key: kv/stalwart-oidc
|
||||||
property: client_secret
|
property: client_secret
|
||||||
---
|
---
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: argocd-oidc-client-secret
|
name: argocd-oidc-client-secret
|
||||||
@@ -100,7 +100,7 @@ spec:
|
|||||||
key: kv/argocd-oidc
|
key: kv/argocd-oidc
|
||||||
property: client_secret
|
property: client_secret
|
||||||
---
|
---
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: freshrss-oidc-client-secret
|
name: freshrss-oidc-client-secret
|
||||||
@@ -118,7 +118,7 @@ spec:
|
|||||||
key: kv/freshrss-oidc
|
key: kv/freshrss-oidc
|
||||||
property: client_secret
|
property: client_secret
|
||||||
---
|
---
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: actualbudget-oidc-client-secret
|
name: actualbudget-oidc-client-secret
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: bolletjestrui-secrets
|
name: bolletjestrui-secrets
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: transip-credentials
|
name: transip-credentials
|
||||||
|
|||||||
+99
-13
@@ -42,29 +42,36 @@ snapshots after every step. Fresh Velero backup before starting.
|
|||||||
- [x] cert-manager 1.15.5 → 1.16.5 → 1.17.4 → 1.18.6 → 1.19.6 → 1.20.4
|
- [x] cert-manager 1.15.5 → 1.16.5 → 1.17.4 → 1.18.6 → 1.19.6 → 1.20.4
|
||||||
(1.18 changes the `rotationPolicy` default to Always)
|
(1.18 changes the `rotationPolicy` default to Always)
|
||||||
|
|
||||||
## 4. External Secrets 0.10.7 → ≥0.17
|
## 4. External Secrets 0.10.7 → 2.13.0
|
||||||
|
|
||||||
- [ ] Move every ExternalSecret and SecretStore to `external-secrets.io/v1`
|
- [x] Every ExternalSecret and the ClusterSecretStore on `external-secrets.io/v1`
|
||||||
- [ ] Upgrade one minor at a time; 0.16 needs v1alpha1 gone from
|
- [x] One minor at a time, 28 hops. The ESO app syncs with server-side apply.
|
||||||
`storedVersions`, 0.17 stops serving v1beta1, 0.19 needs server-side apply
|
At 0.16.2 the old conversion webhook blocked the new CRDs; setting
|
||||||
|
`spec.conversion` to `None` by hand on the three CRDs fixed it
|
||||||
|
- [ ] Later: once every object is stored as v1, drop v1beta1 from the CRDs'
|
||||||
|
`status.storedVersions` (needed before ESO removes v1beta1 entirely)
|
||||||
|
|
||||||
## 5. Argo CD 2.14.2 → 3.4.12
|
## 5. Argo CD 2.14.2 → 3.4.12
|
||||||
|
|
||||||
- [ ] Read every upgrade note from
|
- [x] v3.0.23 → v3.1.16 → v3.2.12 → v3.3.18 → v3.4.12. Label tracking is
|
||||||
[2.14→3.0](https://argo-cd.readthedocs.io/en/stable/operator-manual/upgrading/2.14-3.0/)
|
pinned in `argocd-cm`; our own `resource.exclusions` replaces the 3.0
|
||||||
onward. 3.0 switches to annotation tracking and excludes Endpoints and
|
defaults, so the home-assistant EndpointSlice stays managed.
|
||||||
EndpointSlices by default (affects `home-assistant/service.yaml`); 3.3
|
`argocd-folder` syncs with server-side apply (needed from 3.3)
|
||||||
needs `kubectl apply --server-side --force-conflicts`
|
- [ ] Later: switch to annotation tracking (remove the pin, then sync every
|
||||||
|
app once)
|
||||||
|
|
||||||
## 6. Calico 3.25.1 → 3.29.3
|
## 6. Calico 3.25.1 → 3.29.3
|
||||||
|
|
||||||
- [ ] Apply the manifest microk8s 1.33 ships
|
- [x] Applied microk8s 1.33's manifest (canonical/microk8s branch 1.33,
|
||||||
(`/snap/microk8s/current/upgrade-scripts/000-switch-to-calico/resources/calico.yaml`
|
`upgrade-scripts/000-switch-to-calico/resources/calico.yaml`) with
|
||||||
after the refresh, or the 1.33 branch of canonical/microk8s before it).
|
`kubectl apply --server-side --force-conflicts`. Same calico-node settings
|
||||||
A snap refresh does not upgrade Calico.
|
as before. The manifest is also in
|
||||||
|
`/var/snap/microk8s/current/args/cni-network/cni.yaml`; the 3.25.1 one is
|
||||||
|
saved as `~/cni-3.25.1-backup.yaml` on opti1
|
||||||
|
|
||||||
## 7. microk8s 1.32 → 1.33
|
## 7. microk8s 1.32 → 1.33
|
||||||
|
|
||||||
|
- [x] First fix the k8s-dqlite write stalls, see below
|
||||||
- [ ] Fresh Velero backup and `sudo microk8s dbctl backup`
|
- [ ] Fresh Velero backup and `sudo microk8s dbctl backup`
|
||||||
- [ ] `sudo snap refresh microk8s --channel=1.33/stable`
|
- [ ] `sudo snap refresh microk8s --channel=1.33/stable`
|
||||||
- [ ] `sudo microk8s addons repo update core`
|
- [ ] `sudo microk8s addons repo update core`
|
||||||
@@ -80,3 +87,82 @@ snapshots after every step. Fresh Velero backup before starting.
|
|||||||
|
|
||||||
Out of scope here but due: the immich PostgreSQL 14 cluster reaches end of
|
Out of scope here but due: the immich PostgreSQL 14 cluster reaches end of
|
||||||
life on 2026-11-12.
|
life on 2026-11-12.
|
||||||
|
|
||||||
|
## k8s-dqlite write stalls (found 2026-10-22)
|
||||||
|
|
||||||
|
Lease updates regularly take more than 10s (169 of ~162k since the API
|
||||||
|
server started), so controllers using leader election keep losing it and
|
||||||
|
restarting: snapshot-controller over 1000 times, the CNPG operator,
|
||||||
|
Longhorn's CSI sidecars.
|
||||||
|
|
||||||
|
Cause: the SQLite file behind dqlite is ~709 MB (173k pages of 4 KB) but
|
||||||
|
164k pages are free. The live data is ~35 MB: 3,700 kine rows for 3,006
|
||||||
|
keys, so compaction works. SQLite never returns free pages without VACUUM,
|
||||||
|
and dqlite writes the whole file into a snapshot (~150 MB compressed)
|
||||||
|
every 1,024 commits, i.e. every few minutes. Each snapshot takes about 6s
|
||||||
|
of writing on the Apacer AS350 (DRAM-less SATA SSD, which also holds every
|
||||||
|
Longhorn volume). Commits queue behind it. Outside snapshots fsync takes
|
||||||
|
about 1.2 ms.
|
||||||
|
|
||||||
|
Constraints:
|
||||||
|
- microk8s 1.32 and 1.33 ship k8s-dqlite v1.3.5 (dqlite 1.17 LTS). VACUUM
|
||||||
|
needs dqlite 1.18.2+ (microk8s 1.36). `dbctl restore` writes into the
|
||||||
|
running DB, so it doesn't shrink anything.
|
||||||
|
- microk8s 1.34+ ships k8s-dqlite 1.8.x with a reworked write path; that is
|
||||||
|
where Canonical fixed this class of problem (canonical/microk8s#5153).
|
||||||
|
|
||||||
|
Plan (needs sudo; the API is down for the length of a dqlite restart,
|
||||||
|
workloads keep running). `~/dqlite-tuning.sh` and `~/tuning.yaml` on opti1
|
||||||
|
do the first three steps: `ssh -t ruben@192.168.1.204 ./dqlite-tuning.sh`
|
||||||
|
- [x] `sudo microk8s dbctl backup -o /root/dqlite-pre-tuning-<date>`
|
||||||
|
- [x] Install `backend/tuning.yaml` with
|
||||||
|
`snapshot: {threshold: 8192, trailing: 16384}`: a snapshot every ~25
|
||||||
|
minutes instead of every ~3. Never set trailing alone (threshold then
|
||||||
|
becomes 0). Leaving out the kine intervals keeps their defaults. Costs
|
||||||
|
~0.5 GB more raft log on disk.
|
||||||
|
- [x] `sudo snap restart microk8s.daemon-k8s-dqlite`, then check snapshot
|
||||||
|
spacing in the backend dir and that lease updates stay under 10s.
|
||||||
|
Measured over the first 40 minutes: snapshots 24.7 minutes apart
|
||||||
|
(were ~3), 0 of 9,445 lease updates over 4s, no controller restarts
|
||||||
|
- [ ] Optional: Longhorn backupstore poll interval 300s -> 1800s
|
||||||
|
(backupvolume updates are ~8% of all writes)
|
||||||
|
- Later: microk8s 1.34+ for the new write path and 1.36 for VACUUM, see
|
||||||
|
"On to 1.36" below.
|
||||||
|
|
||||||
|
`--cgroups-per-qos=false`, the likely trigger of the 1.32->1.33 kubelite
|
||||||
|
panic (#5057), is not set in `args/kubelet`.
|
||||||
|
|
||||||
|
## On to 1.36
|
||||||
|
|
||||||
|
microk8s goes up one minor per refresh. 1.34 brings k8s-dqlite 1.8.x with
|
||||||
|
the reworked write path, 1.36 brings dqlite 1.18.2, which can VACUUM the
|
||||||
|
mostly empty database. Before every hop: fresh Velero backup, no
|
||||||
|
deprecated APIs in use (`apiserver_requested_deprecated_apis` metric
|
||||||
|
empty), and the Calico manifest of the target version (canonical/microk8s
|
||||||
|
branch `<version>`, `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`).
|
||||||
|
|
||||||
|
### microk8s 1.34
|
||||||
|
- [ ] Read canonical/microk8s#5568 (LIST stalls with k8s-dqlite on 1.34)
|
||||||
|
- [ ] Offline backup first: k8s-dqlite 1.8 migrates the schema on first
|
||||||
|
start, so 1.33 can't read the database afterwards.
|
||||||
|
`microk8s stop; sudo tar -cf /root/backend-1.33.tar /var/snap/microk8s/current/var/kubernetes/backend; microk8s start`
|
||||||
|
- [ ] `sudo snap refresh microk8s --channel=1.34/stable`, then
|
||||||
|
`sudo microk8s addons repo update core` and Calico
|
||||||
|
|
||||||
|
### Components for 1.35 and 1.36
|
||||||
|
- [ ] Longhorn 1.11.2 → 1.12 → 1.13 (one minor at a time, now allowed on
|
||||||
|
1.34); lift the Renovate hold
|
||||||
|
- [ ] Argo CD 3.4 → 3.5 (3.4 is tested up to 1.35)
|
||||||
|
- [ ] Velero 1.16 → 1.17 → 1.18 (1.17 drops restic, we use kopia)
|
||||||
|
- [ ] Recheck CNPG, External Secrets, the Vault chart and Envoy Gateway
|
||||||
|
against 1.36
|
||||||
|
|
||||||
|
### microk8s 1.35 and 1.36
|
||||||
|
- [ ] `sudo snap refresh microk8s --channel=1.35/stable`, addons repo, Calico
|
||||||
|
- [ ] `sudo snap refresh microk8s --channel=1.36/stable`, addons repo, Calico
|
||||||
|
|
||||||
|
### Shrink the database
|
||||||
|
- [ ] On dqlite 1.18.2+: offline backup, then a supervised VACUUM (the
|
||||||
|
maintainers advise against running it unattended). Expect the file to
|
||||||
|
go from ~709 MB to ~35 MB and snapshots to shrink with it
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: freshrss-oidc-client-secret
|
name: freshrss-oidc-client-secret
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: gatus-ntfy
|
name: gatus-ntfy
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: wordpress-secrets
|
name: wordpress-secrets
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: mbgwp-mariadb-credentials
|
name: mbgwp-mariadb-credentials
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: ntfy-auth
|
name: ntfy-auth
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: ntfy
|
- name: ntfy
|
||||||
image: binwiederhier/ntfy:v2.32.0
|
image: binwiederhier/ntfy:v2.33.1
|
||||||
args: ["serve"]
|
args: ["serve"]
|
||||||
env:
|
env:
|
||||||
- name: NTFY_BASE_URL
|
- name: NTFY_BASE_URL
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: serpbear-secrets
|
name: serpbear-secrets
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
name: vault-oidc-config
|
name: vault-oidc-config
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1
|
||||||
kind: ClusterSecretStore
|
kind: ClusterSecretStore
|
||||||
metadata:
|
metadata:
|
||||||
name: vault-backend
|
name: vault-backend
|
||||||
|
|||||||
Reference in new issue
Block a user