mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 14:29:09 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e45413b06 |
No files matched your search
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: actualbudget-oidc-client-secret
|
||||
|
||||
@@ -14,8 +14,5 @@ spec:
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
# From Argo CD 3.3 the ApplicationSet CRD is too big for client-side
|
||||
# apply's last-applied annotation.
|
||||
- ServerSideApply=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -8,16 +8,14 @@ metadata:
|
||||
spec:
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
# From 0.19 the ESO CRDs are too big for client-side apply.
|
||||
- ServerSideApply=true
|
||||
- CreateNamespace=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
project: default
|
||||
sources:
|
||||
- chart: external-secrets
|
||||
repoURL: https://charts.external-secrets.io
|
||||
targetRevision: 2.13.0
|
||||
targetRevision: 0.10.7
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: external-secrets
|
||||
@@ -6,7 +6,7 @@ namespace: argocd
|
||||
|
||||
# additional resources like ingress rules, cluster and repository secrets.
|
||||
resources:
|
||||
- https://raw.githubusercontent.com/argoproj/argo-cd/refs/tags/v3.4.12/manifests/install.yaml
|
||||
- https://raw.githubusercontent.com/argoproj/argo-cd/refs/tags/v2.14.2/manifests/install.yaml
|
||||
- ingress.yaml
|
||||
- oidc-secret.yaml
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: argocd-oidc-secret
|
||||
|
||||
@@ -7,10 +7,6 @@ metadata:
|
||||
app.kubernetes.io/part-of: argocd
|
||||
data:
|
||||
url: https://argocd.rubenhensen.nl
|
||||
# Argo CD 3.0 defaults to annotation tracking. Stay on labels through the
|
||||
# upgrade so no app loses track of its resources; switching is a
|
||||
# separate step that needs every app synced once.
|
||||
application.resourceTrackingMethod: label
|
||||
oidc.config: |
|
||||
name: Authentik
|
||||
issuer: https://authentik.rubenhensen.nl/application/o/argocd/
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: webhook-es
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: authentik-secrets
|
||||
@@ -24,7 +24,7 @@ spec:
|
||||
key: kv/authentik
|
||||
property: bootstrap_token
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: authentik-postgres-credentials
|
||||
@@ -46,7 +46,7 @@ spec:
|
||||
key: kv/authentik
|
||||
property: postgres_password
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vault-oidc-client-secret
|
||||
@@ -64,7 +64,7 @@ spec:
|
||||
key: kv/vault-oidc
|
||||
property: client_secret
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: stalwart-oidc-client-secret
|
||||
@@ -82,7 +82,7 @@ spec:
|
||||
key: kv/stalwart-oidc
|
||||
property: client_secret
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: argocd-oidc-client-secret
|
||||
@@ -100,7 +100,7 @@ spec:
|
||||
key: kv/argocd-oidc
|
||||
property: client_secret
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: freshrss-oidc-client-secret
|
||||
@@ -118,7 +118,7 @@ spec:
|
||||
key: kv/freshrss-oidc
|
||||
property: client_secret
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: actualbudget-oidc-client-secret
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: bolletjestrui-secrets
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: transip-credentials
|
||||
|
||||
+13
-99
@@ -42,36 +42,29 @@ snapshots after every step. Fresh Velero backup before starting.
|
||||
- [x] cert-manager 1.15.5 → 1.16.5 → 1.17.4 → 1.18.6 → 1.19.6 → 1.20.4
|
||||
(1.18 changes the `rotationPolicy` default to Always)
|
||||
|
||||
## 4. External Secrets 0.10.7 → 2.13.0
|
||||
## 4. External Secrets 0.10.7 → ≥0.17
|
||||
|
||||
- [x] Every ExternalSecret and the ClusterSecretStore on `external-secrets.io/v1`
|
||||
- [x] One minor at a time, 28 hops. The ESO app syncs with server-side apply.
|
||||
At 0.16.2 the old conversion webhook blocked the new CRDs; setting
|
||||
`spec.conversion` to `None` by hand on the three CRDs fixed it
|
||||
- [ ] Later: once every object is stored as v1, drop v1beta1 from the CRDs'
|
||||
`status.storedVersions` (needed before ESO removes v1beta1 entirely)
|
||||
- [ ] Move every ExternalSecret and SecretStore to `external-secrets.io/v1`
|
||||
- [ ] Upgrade one minor at a time; 0.16 needs v1alpha1 gone from
|
||||
`storedVersions`, 0.17 stops serving v1beta1, 0.19 needs server-side apply
|
||||
|
||||
## 5. Argo CD 2.14.2 → 3.4.12
|
||||
|
||||
- [x] v3.0.23 → v3.1.16 → v3.2.12 → v3.3.18 → v3.4.12. Label tracking is
|
||||
pinned in `argocd-cm`; our own `resource.exclusions` replaces the 3.0
|
||||
defaults, so the home-assistant EndpointSlice stays managed.
|
||||
`argocd-folder` syncs with server-side apply (needed from 3.3)
|
||||
- [ ] Later: switch to annotation tracking (remove the pin, then sync every
|
||||
app once)
|
||||
- [ ] Read every upgrade note from
|
||||
[2.14→3.0](https://argo-cd.readthedocs.io/en/stable/operator-manual/upgrading/2.14-3.0/)
|
||||
onward. 3.0 switches to annotation tracking and excludes Endpoints and
|
||||
EndpointSlices by default (affects `home-assistant/service.yaml`); 3.3
|
||||
needs `kubectl apply --server-side --force-conflicts`
|
||||
|
||||
## 6. Calico 3.25.1 → 3.29.3
|
||||
|
||||
- [x] Applied microk8s 1.33's manifest (canonical/microk8s branch 1.33,
|
||||
`upgrade-scripts/000-switch-to-calico/resources/calico.yaml`) with
|
||||
`kubectl apply --server-side --force-conflicts`. Same calico-node settings
|
||||
as before. The manifest is also in
|
||||
`/var/snap/microk8s/current/args/cni-network/cni.yaml`; the 3.25.1 one is
|
||||
saved as `~/cni-3.25.1-backup.yaml` on opti1
|
||||
- [ ] Apply the manifest microk8s 1.33 ships
|
||||
(`/snap/microk8s/current/upgrade-scripts/000-switch-to-calico/resources/calico.yaml`
|
||||
after the refresh, or the 1.33 branch of canonical/microk8s before it).
|
||||
A snap refresh does not upgrade Calico.
|
||||
|
||||
## 7. microk8s 1.32 → 1.33
|
||||
|
||||
- [x] First fix the k8s-dqlite write stalls, see below
|
||||
- [ ] Fresh Velero backup and `sudo microk8s dbctl backup`
|
||||
- [ ] `sudo snap refresh microk8s --channel=1.33/stable`
|
||||
- [ ] `sudo microk8s addons repo update core`
|
||||
@@ -87,82 +80,3 @@ snapshots after every step. Fresh Velero backup before starting.
|
||||
|
||||
Out of scope here but due: the immich PostgreSQL 14 cluster reaches end of
|
||||
life on 2026-11-12.
|
||||
|
||||
## k8s-dqlite write stalls (found 2026-10-22)
|
||||
|
||||
Lease updates regularly take more than 10s (169 of ~162k since the API
|
||||
server started), so controllers using leader election keep losing it and
|
||||
restarting: snapshot-controller over 1000 times, the CNPG operator,
|
||||
Longhorn's CSI sidecars.
|
||||
|
||||
Cause: the SQLite file behind dqlite is ~709 MB (173k pages of 4 KB) but
|
||||
164k pages are free. The live data is ~35 MB: 3,700 kine rows for 3,006
|
||||
keys, so compaction works. SQLite never returns free pages without VACUUM,
|
||||
and dqlite writes the whole file into a snapshot (~150 MB compressed)
|
||||
every 1,024 commits, i.e. every few minutes. Each snapshot takes about 6s
|
||||
of writing on the Apacer AS350 (DRAM-less SATA SSD, which also holds every
|
||||
Longhorn volume). Commits queue behind it. Outside snapshots fsync takes
|
||||
about 1.2 ms.
|
||||
|
||||
Constraints:
|
||||
- microk8s 1.32 and 1.33 ship k8s-dqlite v1.3.5 (dqlite 1.17 LTS). VACUUM
|
||||
needs dqlite 1.18.2+ (microk8s 1.36). `dbctl restore` writes into the
|
||||
running DB, so it doesn't shrink anything.
|
||||
- microk8s 1.34+ ships k8s-dqlite 1.8.x with a reworked write path; that is
|
||||
where Canonical fixed this class of problem (canonical/microk8s#5153).
|
||||
|
||||
Plan (needs sudo; the API is down for the length of a dqlite restart,
|
||||
workloads keep running). `~/dqlite-tuning.sh` and `~/tuning.yaml` on opti1
|
||||
do the first three steps: `ssh -t ruben@192.168.1.204 ./dqlite-tuning.sh`
|
||||
- [x] `sudo microk8s dbctl backup -o /root/dqlite-pre-tuning-<date>`
|
||||
- [x] Install `backend/tuning.yaml` with
|
||||
`snapshot: {threshold: 8192, trailing: 16384}`: a snapshot every ~25
|
||||
minutes instead of every ~3. Never set trailing alone (threshold then
|
||||
becomes 0). Leaving out the kine intervals keeps their defaults. Costs
|
||||
~0.5 GB more raft log on disk.
|
||||
- [x] `sudo snap restart microk8s.daemon-k8s-dqlite`, then check snapshot
|
||||
spacing in the backend dir and that lease updates stay under 10s.
|
||||
Measured over the first 40 minutes: snapshots 24.7 minutes apart
|
||||
(were ~3), 0 of 9,445 lease updates over 4s, no controller restarts
|
||||
- [ ] Optional: Longhorn backupstore poll interval 300s -> 1800s
|
||||
(backupvolume updates are ~8% of all writes)
|
||||
- Later: microk8s 1.34+ for the new write path and 1.36 for VACUUM, see
|
||||
"On to 1.36" below.
|
||||
|
||||
`--cgroups-per-qos=false`, the likely trigger of the 1.32->1.33 kubelite
|
||||
panic (#5057), is not set in `args/kubelet`.
|
||||
|
||||
## On to 1.36
|
||||
|
||||
microk8s goes up one minor per refresh. 1.34 brings k8s-dqlite 1.8.x with
|
||||
the reworked write path, 1.36 brings dqlite 1.18.2, which can VACUUM the
|
||||
mostly empty database. Before every hop: fresh Velero backup, no
|
||||
deprecated APIs in use (`apiserver_requested_deprecated_apis` metric
|
||||
empty), and the Calico manifest of the target version (canonical/microk8s
|
||||
branch `<version>`, `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`).
|
||||
|
||||
### microk8s 1.34
|
||||
- [ ] Read canonical/microk8s#5568 (LIST stalls with k8s-dqlite on 1.34)
|
||||
- [ ] Offline backup first: k8s-dqlite 1.8 migrates the schema on first
|
||||
start, so 1.33 can't read the database afterwards.
|
||||
`microk8s stop; sudo tar -cf /root/backend-1.33.tar /var/snap/microk8s/current/var/kubernetes/backend; microk8s start`
|
||||
- [ ] `sudo snap refresh microk8s --channel=1.34/stable`, then
|
||||
`sudo microk8s addons repo update core` and Calico
|
||||
|
||||
### Components for 1.35 and 1.36
|
||||
- [ ] Longhorn 1.11.2 → 1.12 → 1.13 (one minor at a time, now allowed on
|
||||
1.34); lift the Renovate hold
|
||||
- [ ] Argo CD 3.4 → 3.5 (3.4 is tested up to 1.35)
|
||||
- [ ] Velero 1.16 → 1.17 → 1.18 (1.17 drops restic, we use kopia)
|
||||
- [ ] Recheck CNPG, External Secrets, the Vault chart and Envoy Gateway
|
||||
against 1.36
|
||||
|
||||
### microk8s 1.35 and 1.36
|
||||
- [ ] `sudo snap refresh microk8s --channel=1.35/stable`, addons repo, Calico
|
||||
- [ ] `sudo snap refresh microk8s --channel=1.36/stable`, addons repo, Calico
|
||||
|
||||
### Shrink the database
|
||||
- [ ] On dqlite 1.18.2+: offline backup, then a supervised VACUUM (the
|
||||
maintainers advise against running it unattended). Expect the file to
|
||||
go from ~709 MB to ~35 MB and snapshots to shrink with it
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: freshrss-oidc-client-secret
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: gatus-ntfy
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wordpress-secrets
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: mbgwp-mariadb-credentials
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: ntfy-auth
|
||||
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: ntfy
|
||||
image: binwiederhier/ntfy:v2.33.1
|
||||
image: binwiederhier/ntfy:v2.32.0
|
||||
args: ["serve"]
|
||||
env:
|
||||
- name: NTFY_BASE_URL
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: serpbear-secrets
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vault-oidc-config
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ClusterSecretStore
|
||||
metadata:
|
||||
name: vault-backend
|
||||
|
||||
Reference in new issue
Block a user