mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 14:29:09 +01:00
Compare commits
179
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f894996e6 | ||
|
|
c380bfc0cf | ||
|
|
c3597ef0a6 | ||
|
|
baa8c70b55 | ||
|
|
1efa6c2b93 | ||
|
|
15a92294a6 | ||
|
|
144b9dda2b | ||
|
|
920e54b43c | ||
|
|
665147a691 | ||
|
|
1ba1d5f1a0 | ||
|
|
285a64f980 | ||
|
|
6cf41b0f14 | ||
|
|
5678adfa82 | ||
|
|
2e2d367f63 | ||
|
|
2a1394763d | ||
|
|
86bbf25145 | ||
|
|
92d5147994 | ||
|
|
9e88a92ff7 | ||
|
|
7b12ee2028 | ||
|
|
039b956764 | ||
|
|
d187f55e57 | ||
|
|
a36fc6f348 | ||
|
|
66207de2dd | ||
|
|
a8ccbef274 | ||
|
|
d0ce5ddbec | ||
|
|
c384ae921f | ||
|
|
13b92fff54 | ||
|
|
3834f7a354 | ||
|
|
8110a235a9 | ||
|
|
fa33bf44ae | ||
|
|
d4e99d28c7 | ||
|
|
c8a81e805d | ||
|
|
3fd585de5f | ||
|
|
6c727346f6 | ||
|
|
ed828b426f | ||
|
|
0dc69d34f0 | ||
|
|
30540ea579 | ||
|
|
53d74768ad | ||
|
|
b6462fb3a9 | ||
|
|
bed68c70a8 | ||
|
|
3991a8d726 | ||
|
|
f6186d5a11 | ||
|
|
2ccaea24a8 | ||
|
|
a2abb66fea | ||
|
|
b721c80599 | ||
|
|
09026a5ff0 | ||
|
|
1fcfd3ad61 | ||
|
|
648f99c505 | ||
|
|
20bd3a4bca | ||
|
|
6cba1a9363 | ||
|
|
acfce6645e | ||
|
|
0de86dd65c | ||
|
|
d7ffdc05f6 | ||
|
|
c0f8034e14 | ||
|
|
a15a4f7c75 | ||
|
|
bdafcaf1e0 | ||
|
|
03c3196242 | ||
|
|
b072099cc1 | ||
|
|
e10ca337f1 | ||
|
|
c72b08867f | ||
|
|
d85362ce3b | ||
|
|
018f528fb3 | ||
|
|
714712b862 | ||
|
|
7dc13466cb | ||
|
|
9b77596bcb | ||
|
|
fbf2cb2d92 | ||
|
|
122873ca5c | ||
|
|
71449dd185 | ||
|
|
297d2dffa5 | ||
|
|
46117ee658 | ||
|
|
2a5bbc7f44 | ||
|
|
d7b0372f1a | ||
|
|
0ad6ca816c | ||
|
|
3e252799a4 | ||
|
|
bfd5619e9e | ||
|
|
0b38d8dec8 | ||
|
|
083353fb9d | ||
|
|
23f9c5c60a | ||
|
|
a8bf3d5b64 | ||
|
|
618954ae2c | ||
|
|
44dc85d306 | ||
|
|
b60735f6b0 | ||
|
|
3a57e5be48 | ||
|
|
1611f5d1ef | ||
|
|
5bb1422594 | ||
|
|
68b692a112 | ||
|
|
72f49bed51 | ||
|
|
f4b42757cd | ||
|
|
f529bd99ee | ||
|
|
74dec430b8 | ||
|
|
6fe0acc061 | ||
|
|
bb34b601d9 | ||
|
|
e98668d5fa | ||
|
|
389f911823 | ||
|
|
8f1c6a5b6e | ||
|
|
770631fdc0 | ||
|
|
855317b84d | ||
|
|
d69f2638d1 | ||
|
|
90e552c9f5 | ||
|
|
ad14995ba0 | ||
|
|
c359384ade | ||
|
|
a4fc386ee8 | ||
|
|
0dc0bc4322 | ||
|
|
d843f31d86 | ||
|
|
da23357946 | ||
|
|
b9d88f41b9 | ||
|
|
3c4a8cc8db | ||
|
|
d17e78130d | ||
|
|
22c5a325d6 | ||
|
|
6b2f5855d7 | ||
|
|
ef07307f93 | ||
|
|
aa940134ea | ||
|
|
f86ed09cd8 | ||
|
|
6d2bc379ba | ||
|
|
c25b8883da | ||
|
|
18f9e2efe6 | ||
|
|
5f5f5b01f0 | ||
|
|
6206ab5a74 | ||
|
|
075b3686c3 | ||
|
|
67c21c30d8 | ||
|
|
0c52d8a467 | ||
|
|
f2e4c5aff6 | ||
|
|
d836e4608c | ||
|
|
1f56e67e8b | ||
|
|
cadac4cebf | ||
|
|
bc71061db7 | ||
|
|
7d3a64e5c8 | ||
|
|
e137e79b75 | ||
|
|
67dc4f0783 | ||
|
|
684f536f2f | ||
|
|
1439466af4 | ||
|
|
57514c3c14 | ||
|
|
57851bf43a | ||
|
|
ada909ee02 | ||
|
|
345de16552 | ||
|
|
7396cc3ed6 | ||
|
|
26ca83457c | ||
|
|
4709194483 | ||
|
|
ad2d6fbae5 | ||
|
|
077711472e | ||
|
|
43f741141d | ||
|
|
cbee4955d3 | ||
|
|
a70dff6fdd | ||
|
|
ab0a09a356 | ||
|
|
b2649a682c | ||
|
|
ce3846bcd3 | ||
|
|
96c13c63f7 | ||
|
|
70151b0ef4 | ||
|
|
f5e8b1e633 | ||
|
|
ae2e5ad9fa | ||
|
|
8cf87483d6 | ||
|
|
c5d2ebaefc | ||
|
|
5f8394e0c0 | ||
|
|
396eb2665c | ||
|
|
858904384c | ||
|
|
9014898fc5 | ||
|
|
111a9ddcea | ||
|
|
a8296a514e | ||
|
|
712fa1fe65 | ||
|
|
c3505bc8e3 | ||
|
|
6c6f473f63 | ||
|
|
a5478d60e6 | ||
|
|
88f68cd698 | ||
|
|
f69b790a58 | ||
|
|
71af3a9342 | ||
|
|
348cce6d5d | ||
|
|
abed11dbaa | ||
|
|
86a0c59c6b | ||
|
|
ebf28c1bf1 | ||
|
|
135bca4a83 | ||
|
|
1c6b42b20b | ||
|
|
f3f36c3718 | ||
|
|
399387bfb7 | ||
|
|
48b0873ca3 | ||
|
|
278f6fc55f | ||
|
|
f4e74a9503 | ||
|
|
c270bcc12b | ||
|
|
14e61ee5ee | ||
|
|
45d1ea0d93 |
No files matched your search
@@ -0,0 +1,56 @@
|
||||
# Renders every Argo CD app the way Argo CD does and validates the result
|
||||
# (tools/ci/validate.py), so a broken manifest or a Renovate bump that no
|
||||
# longer renders shows up on the PR instead of in a failed sync.
|
||||
name: validate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: validate-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install helm, kustomize and kubeconform
|
||||
env:
|
||||
# Helm 4.2, like Argo CD 3.5.
|
||||
# renovate: datasource=github-releases depName=helm/helm
|
||||
HELM_VERSION: v4.2.1
|
||||
# renovate: datasource=github-releases depName=kubernetes-sigs/kustomize extractVersion=^kustomize/(?<version>.+)$
|
||||
KUSTOMIZE_VERSION: v5.8.3
|
||||
# renovate: datasource=github-releases depName=yannh/kubeconform
|
||||
KUBECONFORM_VERSION: v0.8.0
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd "$RUNNER_TEMP"
|
||||
curl -sSfLO "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz"
|
||||
curl -sSfL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz.sha256sum" | sha256sum -c -
|
||||
k="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}"
|
||||
curl -sSfLO "$k/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz"
|
||||
curl -sSfL "$k/checksums.txt" | grep " kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz$" | sha256sum -c -
|
||||
c="https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}"
|
||||
curl -sSfLO "$c/kubeconform-linux-amd64.tar.gz"
|
||||
curl -sSfL "$c/CHECKSUMS" | grep " kubeconform-linux-amd64.tar.gz$" | sha256sum -c -
|
||||
mkdir -p bin
|
||||
tar xzf "helm-${HELM_VERSION}-linux-amd64.tar.gz" -C bin --strip-components=1 linux-amd64/helm
|
||||
tar xzf "kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" -C bin kustomize
|
||||
tar xzf kubeconform-linux-amd64.tar.gz -C bin kubeconform
|
||||
echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
|
||||
pip install --quiet pyyaml==6.0.3
|
||||
|
||||
- name: Render and validate
|
||||
run: python3 tools/ci/validate.py
|
||||
@@ -6,3 +6,4 @@ velero-credentials
|
||||
nix-infra-machine/ssh/
|
||||
nix-infra-machine/.env
|
||||
nix-infra-machine/nix/
|
||||
.DS_Store
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://community-charts.github.io/helm-charts
|
||||
targetRevision: 1.8.9
|
||||
targetRevision: 1.9.5
|
||||
chart: actualbudget
|
||||
helm:
|
||||
values: |
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# Actual faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces.
|
||||
# The OIDC login goes through authentik's public address.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 5006
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
sources:
|
||||
- chart: authentik
|
||||
repoURL: https://charts.goauthentik.io
|
||||
targetRevision: 2026.2.1
|
||||
targetRevision: 2026.8.5
|
||||
helm:
|
||||
values: |
|
||||
authentik:
|
||||
@@ -99,6 +99,12 @@ spec:
|
||||
- name: REQUESTS_CA_BUNDLE
|
||||
value: /etc/ssl/certs/ca-certificates.crt
|
||||
server:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 70m
|
||||
memory: 560Mi
|
||||
limits:
|
||||
memory: 1536Mi
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
@@ -127,6 +133,29 @@ spec:
|
||||
- name: blueprint-longhorn
|
||||
configMap:
|
||||
name: authentik-blueprint-longhorn
|
||||
# Directory mounts, not subPath, so changes reach the pods
|
||||
# without a restart. The secrets are optional: a blueprint
|
||||
# that needs a missing one fails until it shows up.
|
||||
- name: blueprint-scouting
|
||||
configMap:
|
||||
name: authentik-blueprint-scouting
|
||||
- name: scouting-secrets
|
||||
projected:
|
||||
sources:
|
||||
- secret:
|
||||
name: immich-oidc-client-secret
|
||||
optional: true
|
||||
items:
|
||||
- key: client_secret
|
||||
path: immich-oidc-client-secret
|
||||
- secret:
|
||||
name: scouts-online-client
|
||||
optional: true
|
||||
items:
|
||||
- key: client_id
|
||||
path: scouts-online-client-id
|
||||
- key: client_secret
|
||||
path: scouts-online-client-secret
|
||||
volumeMounts:
|
||||
- name: blueprint-vault
|
||||
mountPath: /blueprints/custom/vault-oidc.yaml
|
||||
@@ -152,10 +181,21 @@ spec:
|
||||
- name: blueprint-longhorn
|
||||
mountPath: /blueprints/custom/longhorn-oidc.yaml
|
||||
subPath: longhorn-oidc.yaml
|
||||
- name: blueprint-scouting
|
||||
mountPath: /blueprints/custom/scouting
|
||||
- name: scouting-secrets
|
||||
mountPath: /secrets/scouting
|
||||
readOnly: true
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
worker:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 336Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: patched-ca-bundle
|
||||
configMap:
|
||||
@@ -184,6 +224,29 @@ spec:
|
||||
- name: blueprint-longhorn
|
||||
configMap:
|
||||
name: authentik-blueprint-longhorn
|
||||
# Directory mounts, not subPath, so changes reach the pods
|
||||
# without a restart. The secrets are optional: a blueprint
|
||||
# that needs a missing one fails until it shows up.
|
||||
- name: blueprint-scouting
|
||||
configMap:
|
||||
name: authentik-blueprint-scouting
|
||||
- name: scouting-secrets
|
||||
projected:
|
||||
sources:
|
||||
- secret:
|
||||
name: immich-oidc-client-secret
|
||||
optional: true
|
||||
items:
|
||||
- key: client_secret
|
||||
path: immich-oidc-client-secret
|
||||
- secret:
|
||||
name: scouts-online-client
|
||||
optional: true
|
||||
items:
|
||||
- key: client_id
|
||||
path: scouts-online-client-id
|
||||
- key: client_secret
|
||||
path: scouts-online-client-secret
|
||||
volumeMounts:
|
||||
- name: blueprint-vault
|
||||
mountPath: /blueprints/custom/vault-oidc.yaml
|
||||
@@ -209,17 +272,16 @@ spec:
|
||||
- name: blueprint-longhorn
|
||||
mountPath: /blueprints/custom/longhorn-oidc.yaml
|
||||
subPath: longhorn-oidc.yaml
|
||||
- name: blueprint-scouting
|
||||
mountPath: /blueprints/custom/scouting
|
||||
- name: scouting-secrets
|
||||
mountPath: /secrets/scouting
|
||||
readOnly: true
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
postgresql:
|
||||
enabled: false
|
||||
redis:
|
||||
enabled: true
|
||||
master:
|
||||
persistence:
|
||||
storageClass: longhorn
|
||||
size: 2Gi
|
||||
- repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: authentik
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://charts.jetstack.io
|
||||
targetRevision: v1.21.2
|
||||
targetRevision: v1.21.3
|
||||
chart: cert-manager
|
||||
helm:
|
||||
parameters:
|
||||
@@ -34,6 +34,26 @@ spec:
|
||||
- name: patched-ca-bundle
|
||||
mountPath: /etc/ssl/certs
|
||||
readOnly: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
cainjector:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 80Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
webhook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: secrets-folder
|
||||
name: cloudnative-pg-folder
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
@@ -10,12 +10,10 @@ spec:
|
||||
source:
|
||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: secrets
|
||||
path: cloudnative-pg
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: secrets
|
||||
namespace: cnpg-system
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -19,6 +19,17 @@ spec:
|
||||
# upgrades instead of restarting them. Both clusters have a
|
||||
# single instance, so a restart means downtime.
|
||||
ENABLE_INSTANCE_MANAGER_INPLACE_UPDATES: "true"
|
||||
# Ride out dqlite stalls of up to ~40s instead of restarting when a
|
||||
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
|
||||
additionalArgs:
|
||||
- --leader-lease-duration=60
|
||||
- --leader-renew-deadline=40
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cnpg-system
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: default-folder
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: default
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: default
|
||||
syncPolicy:
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -12,7 +12,20 @@ spec:
|
||||
repoURL: docker.io/envoyproxy
|
||||
chart: gateway-helm
|
||||
# v1.9 supports Kubernetes 1.33 to 1.36
|
||||
targetRevision: v1.9.2
|
||||
targetRevision: 1.9.3
|
||||
helm:
|
||||
values: |
|
||||
config:
|
||||
envoyGateway:
|
||||
provider:
|
||||
kubernetes:
|
||||
# Ride out dqlite stalls of up to ~40s instead of restarting
|
||||
# when a lease renewal takes over 10s
|
||||
# (docs/upgrade-k8s-1.33.md).
|
||||
leaderElection:
|
||||
leaseDuration: 60s
|
||||
renewDeadline: 40s
|
||||
retryPeriod: 5s
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: envoy-gateway-system
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: external-secrets-folder
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||
targetRevision: HEAD
|
||||
path: external-secrets
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: external-secrets
|
||||
syncPolicy:
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -17,7 +17,29 @@ spec:
|
||||
sources:
|
||||
- chart: external-secrets
|
||||
repoURL: https://charts.external-secrets.io
|
||||
targetRevision: 2.13.0
|
||||
targetRevision: 2.14.0
|
||||
helm:
|
||||
values: |
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
certController:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
webhook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: external-secrets
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://groundhog2k.github.io/helm-charts/
|
||||
targetRevision: 0.15.0
|
||||
targetRevision: 0.16.7
|
||||
chart: wordpress
|
||||
helm:
|
||||
values: |
|
||||
@@ -101,6 +101,23 @@ spec:
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
|
||||
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
|
||||
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
|
||||
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
|
||||
# crawler asking for 88 pages at once got the container OOMKilled on
|
||||
# 2026-10-25. With 12 workers a burst waits in the listen backlog
|
||||
# instead. Listen is the chart's default (containerPort).
|
||||
apachePortsConfig: |
|
||||
Listen 8000
|
||||
<IfModule mpm_prefork_module>
|
||||
StartServers 4
|
||||
MinSpareServers 2
|
||||
MaxSpareServers 6
|
||||
ServerLimit 12
|
||||
MaxRequestWorkers 12
|
||||
MaxConnectionsPerChild 1000
|
||||
</IfModule>
|
||||
|
||||
# Custom PHP configuration
|
||||
customPhpConfig: |
|
||||
upload_max_filesize = 64M
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: metrics-server
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
chart: metrics-server
|
||||
repoURL: https://kubernetes-sigs.github.io/metrics-server/
|
||||
targetRevision: 3.14.0
|
||||
helm:
|
||||
values: |
|
||||
# Resource usage for kubectl top and for sizing requests and limits.
|
||||
# It keeps everything in memory, so it adds no writes to the SSD.
|
||||
args:
|
||||
# microk8s's kubelet serves a self-signed certificate; its own
|
||||
# metrics-server addon passes the same flag.
|
||||
- --kubelet-insecure-tls
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kube-system
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -0,0 +1,34 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: plugin-barman-cloud
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
chart: plugin-barman-cloud
|
||||
repoURL: https://cloudnative-pg.github.io/charts
|
||||
targetRevision: 0.8.1
|
||||
helm:
|
||||
values: |
|
||||
# CNPG's barman cloud plugin, which backs the PostgreSQL clusters
|
||||
# up to Garage on the NAS (authentik/postgresql-backup.yaml,
|
||||
# immich/postgresql-backup.yaml). It has to run in the operator's
|
||||
# namespace; the chart gets its TLS certificates from cert-manager.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cnpg-system
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -7,16 +7,31 @@ spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
||||
targetRevision: v8.2.1
|
||||
targetRevision: v8.7.0
|
||||
path: deploy/kubernetes/snapshot-controller
|
||||
kustomize:
|
||||
patches:
|
||||
# Ride out dqlite stalls of up to ~40s instead of restarting when a
|
||||
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: snapshot-controller
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/args/-
|
||||
value: --leader-election-lease-duration=60s
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/args/-
|
||||
value: --leader-election-renew-deadline=40s
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/args/-
|
||||
value: --leader-election-retry-period=5s
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kube-system
|
||||
syncPolicy:
|
||||
# Was listed twice, with and without prune; Argo CD kept the last one.
|
||||
automated:
|
||||
selfHeal: true
|
||||
prune: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
automated:
|
||||
selfHeal: true
|
||||
@@ -22,3 +22,4 @@ patches:
|
||||
- path: overlays/argocd-rbac-cm.patch.yaml
|
||||
- path: overlays/argocd-cmd-params-cm.patch.yaml
|
||||
- path: overlays/patched-ca-bundle.patch.yaml
|
||||
- path: overlays/resources.patch.yaml
|
||||
@@ -0,0 +1,114 @@
|
||||
# Requests from what the components used on 2026-10-25 (kubectl top every
|
||||
# 5 minutes; highest value plus 10%). Memory limits at about twice the
|
||||
# highest value seen then or on 2026-10-27, rounded up; no CPU limits.
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: argocd-application-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-application-controller
|
||||
resources:
|
||||
requests:
|
||||
cpu: 150m
|
||||
memory: 704Mi
|
||||
limits:
|
||||
memory: 2560Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-repo-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-repo-server
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-server
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-redis
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 96Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-applicationset-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-applicationset-controller
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-notifications-controller
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: argocd-notifications-controller
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: argocd-dex-server
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: dex
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
+26
-3
@@ -14,7 +14,11 @@ Authentik is deployed as the central identity provider, providing OIDC and LDAP
|
||||
- `blueprint-argocd-oidc.yaml` — OIDC provider for ArgoCD
|
||||
- `blueprint-freshrss-oidc.yaml` — OIDC provider for FreshRSS
|
||||
- `blueprint-actualbudget-proxy.yaml` — Proxy provider for Actual Budget
|
||||
- `blueprint-longhorn-oidc.yaml` — OIDC provider for Longhorn (Envoy Gateway SecurityPolicy)
|
||||
- `blueprint-scouting.yaml` — immich's login with Scouts Online, see docs/immich-scouts-online.md
|
||||
- `blueprint-session-duration.yaml` — Session lifetime of the default authentication flow
|
||||
- **NetworkPolicies** (`networkpolicy.yaml`) — the pods reach each other, DNS and the internet,
|
||||
not the LAN or other namespaces; the worker and the database also reach the API server
|
||||
|
||||
## Session lifetime
|
||||
|
||||
@@ -30,9 +34,23 @@ Apache mod_auth_openidc refreshes on every request.
|
||||
|
||||
## LDAP Outpost
|
||||
|
||||
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by:
|
||||
- **SOGo-mail** (in-cluster) — connects via `ak-outpost-ldap-outpost.authentik.svc.cluster.local:3389`
|
||||
- **Stalwart** (NixOS, external) — connects via `ldap.rubenhensen.nl:389`
|
||||
The LDAP outpost exposes Authentik's user directory over LDAP. It is used by
|
||||
**Stalwart** (NixOS on node001), which connects via `ldap.rubenhensen.nl:389`.
|
||||
The outpost itself talks to the server through `https://authentik.rubenhensen.nl/`.
|
||||
|
||||
Authentik's worker deploys the outpost and keeps its version equal to the
|
||||
server's. After the upgrades to 2026.5 and 2026.8 it didn't follow on its
|
||||
own; this rolls it over:
|
||||
|
||||
```bash
|
||||
kubectl -n authentik exec deploy/authentik-worker -- ak shell -c '
|
||||
from authentik.outposts.models import Outpost
|
||||
from authentik.outposts.tasks import outpost_controller
|
||||
outpost_controller.send(Outpost.objects.get(name="LDAP outpost").pk)'
|
||||
```
|
||||
|
||||
Upgrades go one release at a time (2026.2 → 2026.5 → 2026.8); since 2026.8
|
||||
authentik refuses to skip one.
|
||||
|
||||
### NodePort Service
|
||||
|
||||
@@ -87,3 +105,8 @@ microk8s config > ~/.kube/config
|
||||
```
|
||||
|
||||
**Note:** This must be re-applied after MicroK8s upgrades or cert rotations, as MicroK8s may regenerate the CA without the extension. The fix needs to run on every node in the cluster.
|
||||
|
||||
Checked 2026-10-25: the CA from this fix (`CN=microk8s-ca`, 2026-03-21, valid
|
||||
until 2036) still has the extension after the refreshes to 1.34, 1.35 and
|
||||
1.36. Check with
|
||||
`openssl x509 -in /var/snap/microk8s/current/certs/ca.crt -noout -ext keyUsage`.
|
||||
@@ -38,3 +38,12 @@ data:
|
||||
name: Actual Budget
|
||||
provider: !KeyOf actualbudget-provider
|
||||
meta_launch_url: https://ynab.rubenhensen.nl
|
||||
# Only admins: Scouts Online members get accounts in this authentik
|
||||
# too (for immich) and must not reach personal apps.
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf actualbudget-app
|
||||
attrs:
|
||||
group: !Find [authentik_core.group, [name, "authentik Admins"]]
|
||||
@@ -38,3 +38,12 @@ data:
|
||||
name: ArgoCD
|
||||
provider: !KeyOf argocd-provider
|
||||
meta_launch_url: https://argocd.rubenhensen.nl
|
||||
# Only admins: Scouts Online members get accounts in this authentik
|
||||
# too (for immich) and must not reach personal apps.
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf argocd-app
|
||||
attrs:
|
||||
group: !Find [authentik_core.group, [name, "authentik Admins"]]
|
||||
@@ -40,3 +40,12 @@ data:
|
||||
name: FreshRSS
|
||||
provider: !KeyOf freshrss-provider
|
||||
meta_launch_url: https://rss.rubenhensen.nl
|
||||
# Only admins: Scouts Online members get accounts in this authentik
|
||||
# too (for immich) and must not reach personal apps.
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf freshrss-app
|
||||
attrs:
|
||||
group: !Find [authentik_core.group, [name, "authentik Admins"]]
|
||||
@@ -38,3 +38,12 @@ data:
|
||||
name: Stalwart Mail
|
||||
provider: !KeyOf stalwart-provider
|
||||
meta_launch_url: https://mail.rubenhensen.nl
|
||||
# Only admins: Scouts Online members get accounts in this authentik
|
||||
# too (for immich) and must not reach personal apps.
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf stalwart-app
|
||||
attrs:
|
||||
group: !Find [authentik_core.group, [name, "authentik Admins"]]
|
||||
@@ -0,0 +1,289 @@
|
||||
# Login for the Scouting MBG photos (immich on foto.scoutingmbg.nl).
|
||||
# Mounted as a directory, unlike the other blueprints, so authentik picks up
|
||||
# changes without a restart; secrets come from files in /secrets/scouting
|
||||
# for the same reason (see apps/templates/authentik-helm.yaml).
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: authentik-blueprint-scouting
|
||||
data:
|
||||
immich-oidc.yaml: |
|
||||
version: 1
|
||||
metadata:
|
||||
name: immich OIDC
|
||||
entries:
|
||||
# The source and the login flow below come from scouts-online.yaml.
|
||||
- model: authentik_blueprints.metaapplyblueprint
|
||||
attrs:
|
||||
identifiers:
|
||||
path: custom/scouting/scouts-online.yaml
|
||||
required: true
|
||||
# Filled by the Scouts Online source on every login: every member, and
|
||||
# the members with an active Kaderlid role.
|
||||
- model: authentik_core.group
|
||||
id: mbg-leden
|
||||
state: present
|
||||
identifiers:
|
||||
name: MBG leden
|
||||
- model: authentik_core.group
|
||||
id: mbg-kader
|
||||
state: present
|
||||
identifiers:
|
||||
name: MBG kader
|
||||
# immich syncs both claims on every login. A quota of 0 makes immich
|
||||
# refuse uploads, -1 means unlimited.
|
||||
- model: authentik_providers_oauth2.scopemapping
|
||||
id: immich-scope
|
||||
state: present
|
||||
identifiers:
|
||||
name: "immich: role and quota"
|
||||
attrs:
|
||||
name: "immich: role and quota"
|
||||
scope_name: immich
|
||||
description: Upload rights in the Scouting MBG photos
|
||||
expression: |
|
||||
admin = ak_is_group_member(request.user, name="authentik Admins")
|
||||
uploader = admin or ak_is_group_member(request.user, name="MBG kader")
|
||||
return {
|
||||
"immich_role": "admin" if admin else "user",
|
||||
"immich_quota": -1 if uploader else 0,
|
||||
}
|
||||
- model: authentik_providers_oauth2.oauth2provider
|
||||
id: immich-provider
|
||||
state: present
|
||||
identifiers:
|
||||
name: immich
|
||||
attrs:
|
||||
name: immich
|
||||
authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
|
||||
# Straight on to Scouts Online. Admins with a password log in at
|
||||
# authentik first.
|
||||
authentication_flow: !Find [authentik_flows.flow, [slug, scouts-online-login]]
|
||||
invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
|
||||
client_type: confidential
|
||||
client_id: immich
|
||||
client_secret: !File /secrets/scouting/immich-oidc-client-secret
|
||||
redirect_uris:
|
||||
- matching_mode: strict
|
||||
url: https://foto.scoutingmbg.nl/auth/login
|
||||
# "Link to OAuth" in immich's account settings
|
||||
- matching_mode: strict
|
||||
url: https://foto.scoutingmbg.nl/user-settings
|
||||
- matching_mode: strict
|
||||
url: app.immich:///oauth-callback
|
||||
signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Self-signed Certificate"]]
|
||||
property_mappings:
|
||||
- !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
|
||||
- !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]
|
||||
- !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-profile]]
|
||||
- !KeyOf immich-scope
|
||||
- model: authentik_core.application
|
||||
id: immich-app
|
||||
state: present
|
||||
identifiers:
|
||||
slug: immich
|
||||
attrs:
|
||||
name: Scouting MBG foto's
|
||||
provider: !KeyOf immich-provider
|
||||
meta_launch_url: https://foto.scoutingmbg.nl
|
||||
policy_engine_mode: any
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf immich-app
|
||||
attrs:
|
||||
group: !KeyOf mbg-leden
|
||||
# auth.scoutingmbg.nl: straight to Scouts Online, then on to the
|
||||
# photos. Other domains keep the default brand.
|
||||
- model: authentik_brands.brand
|
||||
state: present
|
||||
identifiers:
|
||||
domain: auth.scoutingmbg.nl
|
||||
attrs:
|
||||
branding_title: Scouting MBG
|
||||
flow_authentication: !Find [authentik_flows.flow, [slug, scouts-online-login]]
|
||||
default_application: !KeyOf immich-app
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 1
|
||||
target: !KeyOf immich-app
|
||||
attrs:
|
||||
group: !Find [authentik_core.group, [name, "authentik Admins"]]
|
||||
scouts-online.yaml: |
|
||||
version: 1
|
||||
metadata:
|
||||
name: Scouts Online
|
||||
entries:
|
||||
- model: authentik_sources_oauth.oauthsourcepropertymapping
|
||||
id: user-mapping
|
||||
state: present
|
||||
identifiers:
|
||||
name: "Scouts Online: user"
|
||||
attrs:
|
||||
name: "Scouts Online: user"
|
||||
expression: |
|
||||
# info is the userinfo of Scouts Online (https://login.scouting.nl/docs/api).
|
||||
# It only holds the roles within the organisations of this connection,
|
||||
# which is just the group.
|
||||
from datetime import date
|
||||
|
||||
today = date.today().isoformat()
|
||||
roles = {role.get("id"): role for role in (info.get("roles") or {}).values()}
|
||||
kader = False
|
||||
for assignment in info.get("role_assignments") or []:
|
||||
start = (assignment.get("start_date") or "")[:10]
|
||||
end = (assignment.get("end_date") or "")[:10]
|
||||
if (start and start > today) or (end and end < today):
|
||||
continue
|
||||
role = roles.get(assignment.get("role_id")) or {}
|
||||
if "Kaderlid" in (role.get("member_type") or []):
|
||||
kader = True
|
||||
|
||||
member_id = info.get("member_id")
|
||||
email = info.get("email") or ""
|
||||
# Family members can share an email address; immich needs one per account.
|
||||
if email and info.get("multiple_accounts"):
|
||||
local, _, domain = email.partition("@")
|
||||
email = f"{local}+{member_id}@{domain}"
|
||||
|
||||
return {
|
||||
"username": f"sol-{member_id}",
|
||||
"email": email,
|
||||
"name": info.get("name") or "",
|
||||
"groups": ["MBG leden", "MBG kader"] if kader else ["MBG leden"],
|
||||
"attributes": {"sol_member_id": member_id},
|
||||
}
|
||||
# The source's flows can only be reached through the source.
|
||||
- model: authentik_policies_expression.expressionpolicy
|
||||
id: sso-only
|
||||
state: present
|
||||
identifiers:
|
||||
name: scouts-online-sso-only
|
||||
attrs:
|
||||
expression: return ak_is_sso_flow
|
||||
# New members become external users: they can use the apps they are
|
||||
# let into, not authentik's own interface.
|
||||
- model: authentik_stages_user_write.userwritestage
|
||||
id: user-write
|
||||
state: present
|
||||
identifiers:
|
||||
name: scouts-online-user-write
|
||||
attrs:
|
||||
user_creation_mode: always_create
|
||||
user_type: external
|
||||
- model: authentik_stages_user_login.userloginstage
|
||||
id: user-login
|
||||
state: present
|
||||
identifiers:
|
||||
name: scouts-online-user-login
|
||||
- model: authentik_flows.flow
|
||||
id: enrollment-flow
|
||||
state: present
|
||||
identifiers:
|
||||
slug: scouts-online-enrollment
|
||||
attrs:
|
||||
name: Scouts Online enrollment
|
||||
title: Scouting MBG
|
||||
designation: enrollment
|
||||
authentication: none
|
||||
- model: authentik_flows.flowstagebinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf enrollment-flow
|
||||
stage: !KeyOf user-write
|
||||
- model: authentik_flows.flowstagebinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 1
|
||||
target: !KeyOf enrollment-flow
|
||||
stage: !KeyOf user-login
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf enrollment-flow
|
||||
policy: !KeyOf sso-only
|
||||
- model: authentik_flows.flow
|
||||
id: authentication-flow
|
||||
state: present
|
||||
identifiers:
|
||||
slug: scouts-online-authentication
|
||||
attrs:
|
||||
name: Scouts Online authentication
|
||||
title: Scouting MBG
|
||||
designation: authentication
|
||||
authentication: none
|
||||
- model: authentik_flows.flowstagebinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf authentication-flow
|
||||
stage: !KeyOf user-login
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf authentication-flow
|
||||
policy: !KeyOf sso-only
|
||||
# SOL only lets in members of the organisations picked in its
|
||||
# connection, and only sends roles within them. The callback URL is
|
||||
# https://auth.scoutingmbg.nl/source/oauth/callback/scouts-online/ (and
|
||||
# authentik.rubenhensen.nl, from before auth.scoutingmbg.nl existed)
|
||||
- model: authentik_sources_oauth.oauthsource
|
||||
id: source
|
||||
state: present
|
||||
identifiers:
|
||||
slug: scouts-online
|
||||
attrs:
|
||||
name: Scouts Online
|
||||
slug: scouts-online
|
||||
enabled: true
|
||||
provider_type: openidconnect
|
||||
consumer_key: !File /secrets/scouting/scouts-online-client-id
|
||||
consumer_secret: !File /secrets/scouting/scouts-online-client-secret
|
||||
authorization_url: https://login.scouting.nl/authorize
|
||||
access_token_url: https://login.scouting.nl/token
|
||||
profile_url: https://login.scouting.nl/oidc/userinfo
|
||||
oidc_well_known_url: https://login.scouting.nl/.well-known/openid-configuration
|
||||
oidc_jwks_url: https://login.scouting.nl/.well-known/openid-jwks.json
|
||||
additional_scopes: membership roles
|
||||
pkce: S256
|
||||
authorization_code_auth_method: basic_auth
|
||||
user_matching_mode: identifier
|
||||
# Links to the groups that immich-oidc.yaml creates
|
||||
group_matching_mode: name_link
|
||||
user_path_template: goauthentik.io/sources/scouts-online
|
||||
enrollment_flow: !KeyOf enrollment-flow
|
||||
authentication_flow: !KeyOf authentication-flow
|
||||
user_property_mappings:
|
||||
- !KeyOf user-mapping
|
||||
# Where immich sends people who aren't logged in. An identification
|
||||
# stage without fields and with one source goes straight to it.
|
||||
- model: authentik_stages_identification.identificationstage
|
||||
id: identification
|
||||
state: present
|
||||
identifiers:
|
||||
name: scouts-online-identification
|
||||
attrs:
|
||||
user_fields: []
|
||||
sources:
|
||||
- !KeyOf source
|
||||
show_source_labels: true
|
||||
- model: authentik_flows.flow
|
||||
id: login-flow
|
||||
state: present
|
||||
identifiers:
|
||||
slug: scouts-online-login
|
||||
attrs:
|
||||
name: Scouts Online login
|
||||
title: Scouting MBG
|
||||
designation: authentication
|
||||
- model: authentik_flows.flowstagebinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf login-flow
|
||||
stage: !KeyOf identification
|
||||
@@ -40,3 +40,12 @@ data:
|
||||
name: Vault
|
||||
provider: !KeyOf vault-provider
|
||||
meta_launch_url: https://vault.rubenhensen.nl
|
||||
# Only admins: Scouts Online members get accounts in this authentik
|
||||
# too (for immich) and must not reach personal apps.
|
||||
- model: authentik_policies.policybinding
|
||||
state: present
|
||||
identifiers:
|
||||
order: 0
|
||||
target: !KeyOf vault-app
|
||||
attrs:
|
||||
group: !Find [authentik_core.group, [name, "authentik Admins"]]
|
||||
@@ -14,3 +14,20 @@ spec:
|
||||
usages:
|
||||
- digital signature
|
||||
- key encipherment
|
||||
---
|
||||
# The login of the Scouting MBG photos (immich), see blueprint-scouting.yaml.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: auth-scoutingmbg-tls
|
||||
spec:
|
||||
secretName: auth-scoutingmbg-tls
|
||||
dnsNames:
|
||||
- auth.scoutingmbg.nl
|
||||
issuerRef:
|
||||
group: cert-manager.io
|
||||
kind: ClusterIssuer
|
||||
name: prod-cluster-issuer
|
||||
usages:
|
||||
- digital signature
|
||||
- key encipherment
|
||||
@@ -153,3 +153,46 @@ spec:
|
||||
remoteRef:
|
||||
key: kv/longhorn-oidc
|
||||
property: client_secret
|
||||
---
|
||||
# Mounted as a file for blueprint-scouting.yaml, not as an env variable.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: immich-oidc-client-secret
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: immich-oidc-client-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: client_secret
|
||||
remoteRef:
|
||||
key: kv/immich-oidc
|
||||
property: client_secret
|
||||
---
|
||||
# The Scouts Online connection (login.scouting.nl, "Beheerde websites"),
|
||||
# mounted as files for blueprint-scouting.yaml.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: scouts-online-client
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: scouts-online-client
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: client_id
|
||||
remoteRef:
|
||||
key: kv/scouts-online
|
||||
property: client_id
|
||||
- secretKey: client_secret
|
||||
remoteRef:
|
||||
key: kv/scouts-online
|
||||
property: client_secret
|
||||
@@ -7,8 +7,12 @@ spec:
|
||||
- name: homelab
|
||||
namespace: envoy-gateway-system
|
||||
sectionName: https-authentik
|
||||
- name: homelab
|
||||
namespace: envoy-gateway-system
|
||||
sectionName: https-auth-scoutingmbg
|
||||
hostnames:
|
||||
- authentik.rubenhensen.nl
|
||||
- auth.scoutingmbg.nl
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: authentik-server
|
||||
@@ -32,3 +36,6 @@ spec:
|
||||
- group: ""
|
||||
kind: Secret
|
||||
name: authentik-tls
|
||||
- group: ""
|
||||
kind: Secret
|
||||
name: auth-scoutingmbg-tls
|
||||
@@ -0,0 +1,169 @@
|
||||
# authentik faces the internet (the login for every member of the group),
|
||||
# so its pods only reach each other, DNS and the internet: not the LAN
|
||||
# (NAS, router) or other namespaces. Scouts Online, the apps' OIDC calls
|
||||
# and the LDAP outpost's connection to the server all use public
|
||||
# addresses. The worker and the CNPG instance manager also reach the API
|
||||
# server (the node, port 16443). Ingress comes from Envoy, the CNPG
|
||||
# operator, the node itself (kubelet probes; 10.1.7.64 is the node's
|
||||
# Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr), and for
|
||||
# the LDAP outpost from anywhere: Stalwart on node001 comes in through the
|
||||
# router and the ldap-outpost-external NodePort.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: server-from-envoy
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: authentik
|
||||
app.kubernetes.io/component: server
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 9000
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: ldap-outpost
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: authentik-outpost-ldap
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- ports:
|
||||
- port: 3389
|
||||
- port: 6636
|
||||
---
|
||||
# The outpost controller deploys and updates the LDAP outpost.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: worker-to-api
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: authentik
|
||||
app.kubernetes.io/component: worker
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
---
|
||||
# The CNPG operator checks each instance on its status port and connects to
|
||||
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
||||
# through the API server. The barman cloud sidecar sends the WAL and the
|
||||
# base backups to Garage on the NAS (postgresql-backup.yaml).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: postgres-cnpg
|
||||
spec:
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: cnpg.io/cluster
|
||||
operator: Exists
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: cnpg-system
|
||||
ports:
|
||||
- port: 8000
|
||||
- port: 5432
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.99/32
|
||||
ports:
|
||||
- port: 3900
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -0,0 +1,94 @@
|
||||
# Backups of authentik's database by CNPG itself, through the barman cloud
|
||||
# plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base backup every
|
||||
# night, and every WAL segment as soon as it is full or 5 minutes old
|
||||
# (archive_timeout). Both go to the bucket cnpg-authentik in Garage on the
|
||||
# NAS (nas/garage), so the database can be restored to any moment of the
|
||||
# last 30 days. Velero copies the data directory while PostgreSQL runs,
|
||||
# which is not a consistent copy.
|
||||
# The key is Garage's cnpg-authentik, in Vault as kv/cnpg-authentik-s3; it
|
||||
# can only reach this bucket.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: authentik-postgres-s3
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: authentik-postgres-s3
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
engineVersion: v2
|
||||
data:
|
||||
ACCESS_KEY_ID: "{{ .access_key_id }}"
|
||||
ACCESS_SECRET_KEY: "{{ .secret_access_key }}"
|
||||
# Garage rejects requests signed for another region
|
||||
REGION: garage
|
||||
data:
|
||||
- secretKey: access_key_id
|
||||
remoteRef:
|
||||
key: kv/cnpg-authentik-s3
|
||||
property: access_key_id
|
||||
- secretKey: secret_access_key
|
||||
remoteRef:
|
||||
key: kv/cnpg-authentik-s3
|
||||
property: secret_access_key
|
||||
---
|
||||
apiVersion: barmancloud.cnpg.io/v1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: authentik-postgres
|
||||
annotations:
|
||||
# The CRD comes with the plugin's own app
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
retentionPolicy: 30d
|
||||
configuration:
|
||||
destinationPath: s3://cnpg-authentik/
|
||||
endpointURL: http://192.168.1.99:3900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: authentik-postgres-s3
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: authentik-postgres-s3
|
||||
key: ACCESS_SECRET_KEY
|
||||
region:
|
||||
name: authentik-postgres-s3
|
||||
key: REGION
|
||||
wal:
|
||||
compression: gzip
|
||||
data:
|
||||
compression: gzip
|
||||
instanceSidecarConfiguration:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
env:
|
||||
# Newer boto3 adds checksums that S3-compatible stores can reject;
|
||||
# the plugin's docs suggest sending them only when required.
|
||||
- name: AWS_REQUEST_CHECKSUM_CALCULATION
|
||||
value: when_required
|
||||
- name: AWS_RESPONSE_CHECKSUM_VALIDATION
|
||||
value: when_required
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: ScheduledBackup
|
||||
metadata:
|
||||
name: authentik-postgres-daily
|
||||
spec:
|
||||
cluster:
|
||||
name: authentik-postgres
|
||||
# 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right
|
||||
# away, so WAL archiving has a base backup to start from.
|
||||
schedule: "0 0 3 * * *"
|
||||
immediate: true
|
||||
backupOwnerReference: self
|
||||
method: plugin
|
||||
pluginConfiguration:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
@@ -6,8 +6,20 @@ spec:
|
||||
instances: 1
|
||||
# Pinned so a new operator default (PostgreSQL 18 since CNPG 1.26.3) can't
|
||||
# start an in-place major upgrade. CNPG filled this in at creation.
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:17.11
|
||||
primaryUpdateStrategy: unsupervised
|
||||
# WAL archiving and base backups to Garage, see postgresql-backup.yaml
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: authentik-postgres
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: authentik
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
# Velero Backup Testing Guide
|
||||
|
||||
**Use `restore-test.sh`.** It restores a few namespaces from a real backup
|
||||
into a throwaway k3s in Docker, from a read-only copy of the bucket, and
|
||||
checks that every volume comes back with the bytes it was backed up with:
|
||||
|
||||
```bash
|
||||
backup-testing/restore-test.sh # newest daily backup; ntfy, freshrss, actual-budget
|
||||
backup-testing/restore-test.sh <backup> <namespace>... # something else
|
||||
```
|
||||
|
||||
How it works and why it copies instead of pointing Velero at the NAS is at
|
||||
the top of the script. The rest of this README describes the older k3d
|
||||
scripts, which predate Envoy Gateway and still expect `*.hensen.io` and
|
||||
sol-auth.
|
||||
|
||||
This guide explains how to test your Velero backups using a temporary K3d cluster.
|
||||
|
||||
## Features
|
||||
|
||||
Executable
+222
@@ -0,0 +1,222 @@
|
||||
#!/bin/bash
|
||||
# Restore a few namespaces from the real Velero backups into a throwaway
|
||||
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
|
||||
# changes:
|
||||
# - one backup (backups/<name>/) and the kopia repositories of the chosen
|
||||
# namespaces (kopia/<namespace>/) are copied read-only from the NAS with
|
||||
# rclone into a Docker volume. A second Velero on the real bucket could
|
||||
# run kopia maintenance on the same repositories as the cluster's own
|
||||
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
|
||||
# ReadOnly backup storage location
|
||||
# - storage class longhorn becomes local-path; only core workload
|
||||
# resources are restored (no routes, certificates, ExternalSecrets or
|
||||
# NetworkPolicies)
|
||||
# - every pod volume restore has to complete with as many bytes as its
|
||||
# backup had, and the restored deployments have to become available
|
||||
#
|
||||
# Usage (on a machine with docker, kubectl and jq):
|
||||
# backup-testing/restore-test.sh [backup] [namespace ...]
|
||||
# Defaults: the newest Completed daily-backup, and ntfy freshrss
|
||||
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
|
||||
#
|
||||
# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
|
||||
# into a temporary directory, and the copied backup holds every Secret of
|
||||
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
|
||||
# cluster (kubeconfig in the printed directory) until you run it with
|
||||
# CLEANUP=1. FROM_VOLUME=<docker volume> takes the bucket from a volume
|
||||
# instead of the NAS (for rehearsals; laid out as <bucket>/backups/...).
|
||||
#
|
||||
# Rehearsed on 2026-10-25 against a backup made in the test cluster itself,
|
||||
# which is how two pitfalls showed up: Velero skips hostPath volumes, and
|
||||
# k3s only provisions local PVs (which it can back up and restore) for a
|
||||
# storage class with the annotation defaultVolumeType: local.
|
||||
set -euo pipefail
|
||||
|
||||
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
|
||||
PORT=${PORT:-16443}
|
||||
K3S=rancher/k3s:v1.34.9-k3s1
|
||||
RCLONE=rclone/rclone:1.75.1
|
||||
HELM=alpine/helm:3.19.0
|
||||
VELERO_CHART=12.2.1
|
||||
AWS_PLUGIN=velero/velero-plugin-for-aws:v1.14.4
|
||||
NET=restore-test VOL=restore-test-s3
|
||||
|
||||
cleanup() {
|
||||
docker rm -f -v restore-k3s restore-s3 >/dev/null 2>&1 || true
|
||||
docker volume rm "$VOL" >/dev/null 2>&1 || true
|
||||
docker network rm "$NET" >/dev/null 2>&1 || true
|
||||
[[ -n ${WORK:-} ]] && rm -rf "$WORK"
|
||||
}
|
||||
if [[ ${CLEANUP:-} == 1 ]]; then
|
||||
cleanup; rm -rf "${TMPDIR:-/tmp}"/restore-test.*; echo "cleaned up"; exit 0
|
||||
fi
|
||||
if docker ps -a --format '{{.Names}}' | grep -qx restore-k3s; then
|
||||
echo "A test cluster from an earlier run exists; run with CLEANUP=1 first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
WORK=$(mktemp -d "${TMPDIR:-/tmp}/restore-test.XXXXXX")
|
||||
chmod 700 "$WORK"
|
||||
[[ ${KEEP:-} == 1 ]] || trap cleanup EXIT
|
||||
|
||||
pk() { kubectl --kubeconfig "$PROD_KUBECONFIG" "$@"; }
|
||||
tk() { kubectl --kubeconfig "$WORK/kubeconfig" "$@"; }
|
||||
# wait_for <seconds> <what> <command...>
|
||||
wait_for() {
|
||||
local limit=$1 what=$2; shift 2
|
||||
local end=$((SECONDS + limit))
|
||||
until "$@" >/dev/null 2>&1; do
|
||||
if ((SECONDS > end)); then echo "timed out waiting for $what" >&2; exit 1; fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
|
||||
# request signed for the wrong region (rclone defaults to us-east-1); MinIO
|
||||
# did so with a bare 400 on HEAD.
|
||||
if [[ -n ${FROM_VOLUME:-} ]]; then
|
||||
BUCKET=${BUCKET:-velero-backups}
|
||||
else
|
||||
bsl=$(pk -n velero get backupstoragelocations default -o json)
|
||||
S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
|
||||
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
|
||||
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
|
||||
fi
|
||||
|
||||
backup=${1:-}
|
||||
shift || true
|
||||
namespaces=("$@")
|
||||
[[ ${#namespaces[@]} -gt 0 ]] || namespaces=(ntfy freshrss actual-budget)
|
||||
if [[ -z $backup ]]; then
|
||||
backup=$(pk -n velero get backups.velero.io -o json | jq -r '[.items[]
|
||||
| select(.metadata.labels["velero.io/schedule-name"] == "daily-backup" and .status.phase == "Completed")]
|
||||
| max_by(.status.startTimestamp).metadata.name')
|
||||
fi
|
||||
echo "== backup $backup, namespaces: ${namespaces[*]}"
|
||||
|
||||
docker network create "$NET" >/dev/null
|
||||
docker volume create "$VOL" >/dev/null
|
||||
if [[ -n ${FROM_VOLUME:-} ]]; then
|
||||
echo "== copy from the volume $FROM_VOLUME"
|
||||
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
|
||||
else
|
||||
echo "== copy from the NAS (read-only)"
|
||||
(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
||||
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
||||
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
||||
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
|
||||
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
||||
copy "backups/$backup"
|
||||
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
||||
rm -f "$WORK/nas.env"
|
||||
fi
|
||||
|
||||
echo "== local S3 and k3s"
|
||||
docker run -d --name restore-s3 --network "$NET" -v "$VOL:/data" "$RCLONE" \
|
||||
serve s3 /data --addr :8080 --auth-key test,restore-test-secret >/dev/null
|
||||
s3ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" restore-s3)
|
||||
docker run -d --name restore-k3s --privileged --network "$NET" -p "127.0.0.1:$PORT:6443" "$K3S" \
|
||||
server --disable traefik --disable metrics-server --tls-san 127.0.0.1 --tls-san restore-k3s >/dev/null
|
||||
node_ready() { docker exec restore-k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; }
|
||||
wait_for 180 "k3s" node_ready
|
||||
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$PORT#" > "$WORK/kubeconfig"
|
||||
docker exec restore-k3s cat /etc/rancher/k3s/k3s.yaml | sed 's#https://127.0.0.1:6443#https://restore-k3s:6443#' > "$WORK/kubeconfig-net"
|
||||
|
||||
echo "== Velero"
|
||||
cat > "$WORK/values.yaml" <<EOF
|
||||
configuration:
|
||||
backupStorageLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
bucket: $BUCKET
|
||||
accessMode: ReadOnly
|
||||
config:
|
||||
region: minio
|
||||
s3ForcePathStyle: "true"
|
||||
s3Url: http://$s3ip:8080
|
||||
volumeSnapshotLocation: []
|
||||
defaultRepoMaintainFrequency: 8760h
|
||||
credentials:
|
||||
secretContents:
|
||||
cloud: |
|
||||
[default]
|
||||
aws_access_key_id=test
|
||||
aws_secret_access_key=restore-test-secret
|
||||
initContainers:
|
||||
- name: velero-plugin-for-aws
|
||||
image: $AWS_PLUGIN
|
||||
volumeMounts:
|
||||
- mountPath: /target
|
||||
name: plugins
|
||||
deployNodeAgent: true
|
||||
snapshotsEnabled: false
|
||||
upgradeCRDs: false
|
||||
EOF
|
||||
docker run --rm --network "$NET" -e HOME=/tmp -e KC="$(base64 < "$WORK/kubeconfig-net")" \
|
||||
-e VALUES="$(base64 < "$WORK/values.yaml")" --entrypoint sh "$HELM" -c '
|
||||
echo "$KC" | base64 -d > /tmp/kc && echo "$VALUES" | base64 -d > /tmp/v.yaml &&
|
||||
helm --kubeconfig /tmp/kc install velero velero --repo https://vmware-tanzu.github.io/helm-charts \
|
||||
--version '"$VELERO_CHART"' -n velero --create-namespace -f /tmp/v.yaml >/dev/null'
|
||||
tk apply -f - >/dev/null <<EOF
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: change-storage-class
|
||||
namespace: velero
|
||||
labels:
|
||||
velero.io/plugin-config: ""
|
||||
velero.io/change-storage-class: RestoreItemAction
|
||||
data:
|
||||
longhorn: local-path
|
||||
EOF
|
||||
wait_for 300 "Velero to sync $backup" tk -n velero get backups.velero.io "$backup"
|
||||
echo " Velero sees $backup"
|
||||
|
||||
echo "== restore"
|
||||
tk apply -f - >/dev/null <<EOF
|
||||
apiVersion: velero.io/v1
|
||||
kind: Restore
|
||||
metadata:
|
||||
name: restore-test
|
||||
namespace: velero
|
||||
spec:
|
||||
backupName: $backup
|
||||
includedNamespaces: [$(IFS=,; echo "${namespaces[*]}")]
|
||||
# PersistentVolumes have to be included: only then does Velero clear the
|
||||
# PVC's volumeName. It doesn't restore a PV whose data comes from kopia;
|
||||
# k3s provisions a new one.
|
||||
includedResources: [namespaces, persistentvolumeclaims, persistentvolumes, pods, deployments,
|
||||
replicasets, statefulsets, services, configmaps, secrets, serviceaccounts]
|
||||
restorePVs: true
|
||||
EOF
|
||||
restore_done() { [[ $(tk -n velero get restore restore-test -o jsonpath='{.status.phase}') =~ ^(Completed|PartiallyFailed|Failed|FailedValidation)$ ]]; }
|
||||
wait_for 1800 "the restore" restore_done
|
||||
phase=$(tk -n velero get restore restore-test -o jsonpath='{.status.phase}')
|
||||
echo " restore: $phase"
|
||||
|
||||
echo "== check"
|
||||
failed=0
|
||||
expected=$(docker run --rm -v "$VOL:/data" alpine:3.22 zcat "/data/$BUCKET/backups/$backup/$backup-podvolumebackups.json.gz" |
|
||||
jq -r --arg ns "$(IFS=,; echo "${namespaces[*]}")" '.[] | select(.spec.pod.namespace as $p | $ns | split(",") | index($p))
|
||||
| "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0)"' | sort -u)
|
||||
restored=$(tk -n velero get podvolumerestores.velero.io -o json |
|
||||
jq -r '.items[] | "\(.spec.pod.namespace)/\(.spec.volume) \(.status.progress.totalBytes // 0) \(.status.phase)"' | sort -u)
|
||||
while read -r vol bytes; do
|
||||
[[ -z $vol ]] && continue
|
||||
got=$(awk -v v="$vol" '$1 == v {print $2, $3}' <<<"$restored")
|
||||
if [[ $got == "$bytes Completed" ]]; then
|
||||
echo " ok $vol ($bytes bytes)"
|
||||
else
|
||||
echo " FAILED $vol: backed up $bytes bytes, restore: ${got:-none}"; failed=1
|
||||
fi
|
||||
done <<<"$expected"
|
||||
for ns in "${namespaces[@]}"; do
|
||||
if tk -n "$ns" wait --for=condition=Available deployment --all --timeout=300s >/dev/null 2>&1; then
|
||||
echo " ok $ns: deployments available"
|
||||
else
|
||||
echo " FAILED $ns: deployments not available"; tk -n "$ns" get pods; failed=1
|
||||
fi
|
||||
done
|
||||
[[ $phase == Completed ]] || failed=1
|
||||
[[ ${KEEP:-} == 1 ]] && echo "Kept; kubeconfig: $WORK/kubeconfig. Remove with CLEANUP=1 $0"
|
||||
if [[ $failed == 0 ]]; then echo "== restore test passed"; else echo "== restore test FAILED"; exit 1; fi
|
||||
@@ -1,381 +0,0 @@
|
||||
#!/bin/bash
|
||||
# test-velero-backup.sh - Automated script to test Velero backups
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration - customize these variables
|
||||
BACKUP_NAME="" # Will list backups if empty
|
||||
S3_BUCKET="velero-backups"
|
||||
S3_REGION="minio"
|
||||
S3_URL="http://192.168.1.99:9000" # Your NAS MinIO server
|
||||
S3_ACCESS_KEY="" # Will be filled from credentials
|
||||
S3_SECRET_KEY="" # Will be filled from credentials
|
||||
ORIGINAL_NAMESPACE="" # Namespace in the original backup
|
||||
TEST_NAMESPACE="default"
|
||||
DEBUG=false # Set to true for verbose debugging
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--backup-name)
|
||||
BACKUP_NAME="$2"
|
||||
shift 2
|
||||
;;
|
||||
--s3-access-key)
|
||||
S3_ACCESS_KEY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--s3-secret-key)
|
||||
S3_SECRET_KEY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--s3-bucket)
|
||||
S3_BUCKET="$2"
|
||||
shift 2
|
||||
;;
|
||||
--s3-region)
|
||||
S3_REGION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--s3-url)
|
||||
S3_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--original-namespace)
|
||||
ORIGINAL_NAMESPACE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--test-namespace)
|
||||
TEST_NAMESPACE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--debug)
|
||||
DEBUG=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [options]"
|
||||
echo "Options:"
|
||||
echo " --backup-name NAME Name of the Velero backup to restore"
|
||||
echo " --s3-bucket BUCKET S3 bucket containing backups"
|
||||
echo " --s3-region REGION S3 region"
|
||||
echo " --s3-url URL S3 endpoint URL"
|
||||
echo " --s3-access-key KEY S3 access key"
|
||||
echo " --s3-secret-key KEY S3 secret key"
|
||||
echo " --original-namespace NS Namespace in the original backup"
|
||||
echo " --test-namespace NS Namespace for restoration (default: default)"
|
||||
echo " --debug Enable debug output"
|
||||
echo " --help Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Verify required variables
|
||||
if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" || -z "$ORIGINAL_NAMESPACE" || -z "$TEST_NAMESPACE" ]]; then
|
||||
echo "Error: S3 configuration is incomplete"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to clean up resources
|
||||
cleanup() {
|
||||
echo "Cleaning up resources..."
|
||||
k3d cluster delete backup-test || true
|
||||
echo "Cleanup complete"
|
||||
}
|
||||
|
||||
# Function for debugging output
|
||||
debug() {
|
||||
if [[ "$DEBUG" == "true" ]]; then
|
||||
echo "[DEBUG] $*"
|
||||
fi
|
||||
}
|
||||
|
||||
# Register cleanup function to run on exit (ctrl+c, etc.)
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "=== Starting Velero backup test process ==="
|
||||
|
||||
# 1. Create K3d cluster
|
||||
echo "Creating K3d cluster..."
|
||||
k3d cluster create backup-test \
|
||||
--api-port 6443 \
|
||||
--port "80:80@loadbalancer" \
|
||||
--port "443:443@loadbalancer" \
|
||||
--agents 0 \
|
||||
--k3s-arg "--disable=traefik@server:0" \
|
||||
--wait
|
||||
# -v /mnt/longhorn:/var/lib/longhorn:shared@all \
|
||||
# -i hebury/k3s:v1.32.2-k3s1 \
|
||||
# --wait
|
||||
|
||||
# Verify cluster is running
|
||||
kubectl config use-context k3d-backup-test
|
||||
kubectl get nodes
|
||||
|
||||
# 2. Install Longhorn
|
||||
# echo "Installing Longhorn..."
|
||||
# kubectl create namespace longhorn-system
|
||||
|
||||
# helm repo add longhorn https://charts.longhorn.io
|
||||
# helm repo update
|
||||
|
||||
# helm install longhorn longhorn/longhorn \
|
||||
# --namespace longhorn-system \
|
||||
# --set persistence.defaultClassReplicaCount=1 \
|
||||
# --set defaultSettings.backupTarget="" \
|
||||
# --set defaultSettings.defaultReplicaCount=1
|
||||
|
||||
# echo "Waiting for Longhorn to be ready..."
|
||||
# kubectl -n longhorn-system rollout status deployment/longhorn-ui
|
||||
# kubectl -n longhorn-system rollout status deployment/longhorn-driver-deployer
|
||||
# kubectl -n longhorn-system rollout status daemonset/longhorn-manager
|
||||
|
||||
# 3. Install Velero with S3 provider
|
||||
echo "Installing Velero..."
|
||||
cat > velero-credentials <<EOF
|
||||
[default]
|
||||
aws_access_key_id=$S3_ACCESS_KEY
|
||||
aws_secret_access_key=$S3_SECRET_KEY
|
||||
EOF
|
||||
|
||||
helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
|
||||
helm repo update
|
||||
|
||||
debug "Installing Velero with S3 configuration:"
|
||||
debug " Bucket: $S3_BUCKET"
|
||||
debug " Region: $S3_REGION"
|
||||
debug " URL: $S3_URL"
|
||||
|
||||
helm install velero vmware-tanzu/velero \
|
||||
--namespace velero \
|
||||
--create-namespace \
|
||||
--set-file credentials.secretContents.cloud=velero-credentials \
|
||||
--set configuration.backupStorageLocation[0].provider=aws \
|
||||
--set configuration.backupStorageLocation[0].name=default \
|
||||
--set configuration.backupStorageLocation[0].bucket=$S3_BUCKET \
|
||||
--set configuration.backupStorageLocation[0].config.region=$S3_REGION \
|
||||
--set configuration.backupStorageLocation[0].config.s3ForcePathStyle=true \
|
||||
--set configuration.backupStorageLocation[0].config.s3Url=$S3_URL \
|
||||
--set configuration.volumeSnapshotLocation[0].name=default \
|
||||
--set configuration.volumeSnapshotLocation[0].provider=aws \
|
||||
--set configuration.volumeSnapshotLocation[0].config.region=$S3_REGION \
|
||||
--set configuration.features=EnableCSI \
|
||||
--set initContainers[0].name=velero-plugin-for-aws \
|
||||
--set initContainers[0].image=velero/velero-plugin-for-aws:v1.11.1 \
|
||||
--set initContainers[0].volumeMounts[0].mountPath=/target \
|
||||
--set initContainers[0].volumeMounts[0].name=plugins \
|
||||
--set deployNodeAgent=true \
|
||||
--set nodeAgent.containerSecurityContext.privileged=true \
|
||||
--wait
|
||||
|
||||
rm velero-credentials
|
||||
|
||||
# Wait for Velero to be ready
|
||||
echo "Waiting for Velero to be fully ready..."
|
||||
kubectl -n velero rollout status deployment/velero --timeout=120s
|
||||
|
||||
echo "Install change-storage-class.yaml"
|
||||
kubectl apply -f change-storage-class.yaml
|
||||
|
||||
# Check if velero CLI is installed and functioning
|
||||
if ! command -v velero &> /dev/null; then
|
||||
echo "Error: Velero CLI is not installed or not in PATH"
|
||||
echo "Please install the Velero CLI by following instructions at: https://velero.io/docs/main/basic-install/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify Velero is connected to the storage location
|
||||
echo "Checking Velero backup storage location..."
|
||||
kubectl -n velero get backupstoragelocation default -o jsonpath='{.status.phase}' | grep -q "Available" || {
|
||||
echo "Warning: Backup storage location is not available, it may take a few minutes to initialize"
|
||||
echo "Storage location status:"
|
||||
kubectl -n velero get backupstoragelocation default -o yaml
|
||||
|
||||
# Wait a bit longer for the location to become available
|
||||
echo "Waiting 30 seconds for backup storage location to become available..."
|
||||
sleep 30
|
||||
}
|
||||
|
||||
# 4. List backups
|
||||
echo "Checking for available backups..."
|
||||
if ! velero_output=$(velero backup get 2>&1); then
|
||||
echo "Error listing backups: $velero_output"
|
||||
|
||||
# Try to verify S3 connectivity
|
||||
echo "Checking S3 connectivity..."
|
||||
|
||||
# Install AWS CLI if needed
|
||||
if ! command -v aws &> /dev/null; then
|
||||
echo "AWS CLI not found, skipping S3 connectivity test"
|
||||
else
|
||||
# Create temporary AWS profile
|
||||
mkdir -p ~/.aws
|
||||
cat > ~/.aws/credentials <<EOF
|
||||
[velero-test]
|
||||
aws_access_key_id=$S3_ACCESS_KEY
|
||||
aws_secret_access_key=$S3_SECRET_KEY
|
||||
EOF
|
||||
|
||||
cat > ~/.aws/config <<EOF
|
||||
[profile velero-test]
|
||||
region=$S3_REGION
|
||||
output=json
|
||||
EOF
|
||||
|
||||
# Try to list bucket contents
|
||||
echo "Testing S3 connection to bucket $S3_BUCKET..."
|
||||
if aws --profile velero-test --endpoint-url $S3_URL s3 ls s3://$S3_BUCKET/; then
|
||||
echo "S3 connection successful, but Velero couldn't list backups."
|
||||
echo "This might indicate that there are no backups in the bucket or the backups are in a different format than expected."
|
||||
else
|
||||
echo "Error connecting to S3. Please check your credentials and connectivity to $S3_URL"
|
||||
fi
|
||||
|
||||
# Cleanup AWS files
|
||||
rm ~/.aws/credentials ~/.aws/config
|
||||
fi
|
||||
|
||||
if [[ -z "$BACKUP_NAME" ]]; then
|
||||
echo "No backup name specified and no backups found. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Available backups:"
|
||||
echo "$velero_output"
|
||||
|
||||
if [[ -z "$BACKUP_NAME" ]]; then
|
||||
echo "Specify a backup name using --backup-name and run again."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Create test namespace if needed
|
||||
if [[ "$TEST_NAMESPACE" != "default" ]]; then
|
||||
kubectl create namespace $TEST_NAMESPACE || true
|
||||
fi
|
||||
|
||||
# 6. Restore backup
|
||||
echo "Restoring backup: $BACKUP_NAME"
|
||||
if [[ -z "$ORIGINAL_NAMESPACE" ]]; then
|
||||
# If no namespace specified, restore everything
|
||||
velero restore create --from-backup $BACKUP_NAME \
|
||||
--wait
|
||||
else
|
||||
# Restore specific namespace with mapping
|
||||
velero restore create --from-backup $BACKUP_NAME \
|
||||
--namespace-mappings $ORIGINAL_NAMESPACE:$TEST_NAMESPACE \
|
||||
--include-namespaces $ORIGINAL_NAMESPACE \
|
||||
--exclude-namespaces kube-system,kube-public,kube-node-lease,velero \
|
||||
--exclude-resources certificates.cert-manager.io \
|
||||
--wait
|
||||
fi
|
||||
|
||||
# 7. Check restore status
|
||||
echo "Checking restore status:"
|
||||
velero restore get
|
||||
|
||||
# Wait for pods to be ready
|
||||
echo "Waiting for pods to be ready..."
|
||||
kubectl wait --for=condition=ready pod --all -n $TEST_NAMESPACE --timeout=600s || true
|
||||
|
||||
# 8. Run validation script (create it first if it doesn't exist)
|
||||
# if [[ ! -f ./validate-restore.sh ]]; then
|
||||
# echo "Creating validation script..."
|
||||
# cat > validate-restore.sh <<'EOF'
|
||||
# #!/bin/bash
|
||||
# # validate-restore.sh - Script to validate Velero backup restoration
|
||||
|
||||
# set -e
|
||||
# echo "Starting validation of restored resources..."
|
||||
|
||||
# # Get namespace from command line or use default
|
||||
# NAMESPACE="${1:-default}"
|
||||
|
||||
# # Check all deployments
|
||||
# echo "Checking deployments..."
|
||||
# deployments=$(kubectl get deployment -n $NAMESPACE -o name)
|
||||
# if [[ -z "$deployments" ]]; then
|
||||
# echo "❌ No deployments found in namespace $NAMESPACE"
|
||||
# else
|
||||
# echo "Found deployments: $deployments"
|
||||
|
||||
# # Check if pods are running for each deployment
|
||||
# for deployment in $deployments; do
|
||||
# name=$(echo $deployment | cut -d'/' -f2)
|
||||
# echo -n "Checking deployment $name: "
|
||||
|
||||
# # Check if pods are running
|
||||
# ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
|
||||
# if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
|
||||
# echo "✅ $ready_replicas pods ready"
|
||||
# else
|
||||
# echo "❌ No pods running"
|
||||
# fi
|
||||
# done
|
||||
# fi
|
||||
|
||||
# # Check all services
|
||||
# echo "Checking services..."
|
||||
# services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
|
||||
# if [[ -z "$services" ]]; then
|
||||
# echo "❌ No services found in namespace $NAMESPACE"
|
||||
# else
|
||||
# echo "Found services: $services"
|
||||
|
||||
# # Check if services have endpoints
|
||||
# for service in $services; do
|
||||
# name=$(echo $service | cut -d'/' -f2)
|
||||
# echo -n "Checking service $name: "
|
||||
|
||||
# # Check if service has endpoints
|
||||
# endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
|
||||
# if [ -n "$endpoints" ]; then
|
||||
# echo "✅ Has endpoints"
|
||||
# else
|
||||
# echo "❌ No endpoints"
|
||||
# fi
|
||||
# done
|
||||
# fi
|
||||
|
||||
# # Check PVCs
|
||||
# echo "Checking PVCs..."
|
||||
# pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
|
||||
# if [[ -z "$pvcs" ]]; then
|
||||
# echo "No PVCs found in namespace $NAMESPACE"
|
||||
# else
|
||||
# pvc_count=$(echo "$pvcs" | wc -l)
|
||||
# echo "Found $pvc_count PVCs"
|
||||
|
||||
# # Check if PVCs are bound
|
||||
# bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
|
||||
# echo "$bound_count/$pvc_count PVCs are bound"
|
||||
|
||||
# if [ "$bound_count" -ne "$pvc_count" ]; then
|
||||
# echo "❌ Not all PVCs are bound"
|
||||
# else
|
||||
# echo "✅ All PVCs are bound"
|
||||
# fi
|
||||
# fi
|
||||
|
||||
# echo "Validation complete!"
|
||||
# exit 0
|
||||
# EOF
|
||||
# chmod +x validate-restore.sh
|
||||
# fi
|
||||
|
||||
echo "Running validation tests..."
|
||||
./validate-restore.sh $TEST_NAMESPACE
|
||||
|
||||
echo ""
|
||||
echo "=== Backup test process completed ==="
|
||||
echo "The cluster will be deleted when you close this terminal or press Ctrl+C"
|
||||
echo "To access the cluster manually, use: kubectl config use-context k3d-backup-test"
|
||||
echo ""
|
||||
|
||||
# Keep the script running until user terminates it
|
||||
read -p "Press Enter to clean up and exit..."
|
||||
@@ -0,0 +1,82 @@
|
||||
# Bolletjestrui faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 3000
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -0,0 +1,73 @@
|
||||
# cert-manager's three pods reach each other, DNS and the API server (the
|
||||
# node, port 16443), and take connections only from each other and the
|
||||
# node: the API server calls the webhook from there and the kubelet probes
|
||||
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||
# projectcalico.org/IPv4VXLANTunnelAddr). Nothing scrapes the metrics on
|
||||
# 9402. The HTTP-01 solver pods run in each Certificate's own namespace
|
||||
# (acme-http01-solver in <namespace>/networkpolicy.yaml), not here.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
---
|
||||
# The controller talks to Let's Encrypt (443), and before it lets Let's
|
||||
# Encrypt check a challenge it fetches http://<host>/.well-known/... itself
|
||||
# (80). Every certificate's host has a public address, node001's or the
|
||||
# home router's (checked 2026-10-26), so the LAN stays out.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: controller-to-internet
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: cert-manager
|
||||
app.kubernetes.io/component: controller
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
ports:
|
||||
- port: 80
|
||||
- port: 443
|
||||
@@ -0,0 +1,56 @@
|
||||
# The CloudNativePG operator reaches DNS, the API server (the node, port
|
||||
# 16443) and the PostgreSQL instances it runs in other namespaces: the
|
||||
# instance manager's status port (8000) and PostgreSQL (5432), which the
|
||||
# instances' own policies let in (postgres-cnpg in authentik and immich).
|
||||
# Besides its own namespace it takes connections only from the node: the
|
||||
# API server calls its webhook (9443) from there and the kubelet probes
|
||||
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||
# projectcalico.org/IPv4VXLANTunnelAddr). The chart's Application
|
||||
# (apps/templates/cloudnativepg-helm.yaml) only installs the chart, so
|
||||
# this folder gets an Application of its own, cloudnative-pg-folder.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- namespaceSelector: {}
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: cnpg.io/cluster
|
||||
operator: Exists
|
||||
ports:
|
||||
- port: 8000
|
||||
- port: 5432
|
||||
@@ -0,0 +1,14 @@
|
||||
# Nothing runs in default (2026-10-26: only the kubernetes Service, whose
|
||||
# endpoint is the API server on the node, which this doesn't touch). A pod
|
||||
# started here, by `kubectl run` or a manifest without a namespace, gets
|
||||
# no network at all, instead of reaching the LAN, Vault and every app.
|
||||
# Anything that needs a network gets a namespace and a policy of its own.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
+1
-1
@@ -13,7 +13,7 @@ spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: sync
|
||||
image: python:3.12-alpine
|
||||
image: python:3.15-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
|
||||
@@ -7,6 +7,7 @@ resources:
|
||||
- external-secret.yaml
|
||||
- rbac.yaml
|
||||
- cronjob.yaml
|
||||
- networkpolicy.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: dns-sync-script
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# The only pod here is the dns-sync Job (a PostSync hook): it installs its
|
||||
# Python packages from PyPI and pushes domains/ to TransIP's API, so it
|
||||
# reaches DNS and the internet (not the LAN, the API server or other
|
||||
# namespaces). Nothing connects to it.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
ports:
|
||||
- port: 443
|
||||
@@ -0,0 +1,135 @@
|
||||
# immich: log in with Scouts Online
|
||||
|
||||
foto.scoutingmbg.nl sits behind sol-auth, an Apache with mod_auth_openid
|
||||
that lets any Scouting Nederland account through (OpenID 2, which Scouting
|
||||
Nederland is phasing out). Behind it, people use two shared immich
|
||||
accounts: `fotoupload@` owns all 45,703 photos and 504 albums, `foto@`
|
||||
views them (its password is on the login page).
|
||||
|
||||
Goal: everyone logs in with their own Scouts Online 3 account and gets an
|
||||
immich account of their own; kaderleden can upload, everyone else can only
|
||||
view, and everything is shared with everyone automatically.
|
||||
|
||||
## Building blocks
|
||||
|
||||
- **Scouts Online 3** (login.scouting.nl) is an OIDC provider. A connection
|
||||
is registered under "Beheerde websites" by someone with the Webmaster
|
||||
role in SOL 3, and only members of the organisations picked there can
|
||||
log in. The `roles` scope returns the member's organisation units
|
||||
(`Board`, `Game Section` with speltak, committees, ...), roles
|
||||
(`member_type` `Jeugdlid`/`Kaderlid`, a category such as
|
||||
`Leidinggevende`) and role assignments with start and end dates, only
|
||||
from the userinfo endpoint and nested (https://login.scouting.nl/docs/api).
|
||||
- **immich** can't read that: it takes the claims `immich_quota` (GiB; `0`
|
||||
blocks uploads, `-1` is unlimited) and `immich_role` (`user`/`admin`)
|
||||
and syncs both on every login. It links an OAuth login to an existing
|
||||
account by email, or creates one (auto register).
|
||||
- **immich has no user groups**; albums are shared per user, as viewer or
|
||||
editor. The owner and editors of an album may add users and change
|
||||
their roles. Search only covers your own photos and those of partners
|
||||
you show in your timeline.
|
||||
|
||||
## Design
|
||||
|
||||
- **authentik** (on auth.scoutingmbg.nl, with a brand "Scouting MBG" that
|
||||
logs in through Scouts Online and then sends people to the photos)
|
||||
brokers the login: a
|
||||
Scouts Online source turns the roles into the groups `MBG leden` and
|
||||
`MBG kader` (active `Kaderlid` role) on every login, and the immich
|
||||
provider turns those into `immich_quota` and `immich_role`
|
||||
(`authentik/blueprint-scouting.yaml`). Members get `external` users;
|
||||
every personal app in this authentik is bound to `authentik Admins`.
|
||||
- **immich** gets its settings from a config file rendered by ESO
|
||||
(`immich/immich-config.yaml`), OAuth with auto register and, after the
|
||||
switch, auto launch and no password login.
|
||||
- **share-sync** (`immich/share-sync.yaml`), a CronJob every 10 minutes:
|
||||
every user into every album of the group account (`fotoupload@`), as
|
||||
editor or viewer by quota; the group account's library as a partner of
|
||||
everyone, shown in the timeline; album invitations for albums older than
|
||||
a week hidden; sessions of OAuth users older than 30 days ended. Albums
|
||||
that kaderleden create get the group account as editor first (in the
|
||||
database, since the API only lets an album's owner or editors share
|
||||
it), so they go to everyone too; names starting with "Privé"/"Prive"
|
||||
are skipped. Photos outside albums stay with their uploader.
|
||||
|
||||
## Steps
|
||||
|
||||
A. immich OAuth through authentik, next to the password login
|
||||
- [x] Vault `kv/immich-oidc`: `client_secret` (random, e.g. `openssl rand -hex 32`)
|
||||
- [x] Push the authentik and immich changes. Done 2026-10-24: both rolled
|
||||
over without downtime. authentik only found the new blueprint after
|
||||
`blueprints_discovery.send()` in `ak shell`; the claims came out as
|
||||
admin/-1 for authentik Admins, user/-1 for MBG kader, user/0 for MBG
|
||||
leden, and other users get no access
|
||||
- [x] Test with an authentik admin account. The login created a new immich
|
||||
admin (`ik@rubenhensen.nl`, the authentik email) instead of linking the
|
||||
old one (`ruben.hensen@protonmail.com`): linking an existing account
|
||||
takes a password login first, then "Link to OAuth" in its settings
|
||||
|
||||
B. Scouts Online source
|
||||
- [x] SOL connection, by someone with the Webmaster role: redirect URI
|
||||
`https://authentik.rubenhensen.nl/source/oauth/callback/scouts-online/`,
|
||||
post logout `https://foto.scoutingmbg.nl/`, scopes E-mail, Persoonlijk,
|
||||
Lidmaatschap, Rollen en lidmaatschappen; only the group, without
|
||||
sub-organisations; confidential client with PKCE required
|
||||
- [x] Vault `kv/scouts-online`: `client_id`, `client_secret`
|
||||
- [x] Source, enrollment and authentication flows, and a login flow that
|
||||
goes straight to Scouts Online (an identification stage with no fields
|
||||
and one source redirects on its own). Done 2026-10-24; the user mapping
|
||||
was tested beforehand with made-up userinfo (active and ended Kaderlid
|
||||
roles, a shared family email, no roles). A kaderlid's login made an
|
||||
external authentik user `sol-<member id>` in MBG leden and MBG kader, and
|
||||
an immich user with unlimited quota. Use a private window to test: an
|
||||
existing authentik session skips the login flow
|
||||
- [ ] Test with a jeugdlid (quota 0, so no uploads)
|
||||
|
||||
C. Switch
|
||||
- [x] Vault `kv/immich-share`: `api_key` of `fotoupload@` (album.read,
|
||||
albumUser.create, albumUser.update, partner.read, partner.create);
|
||||
share-sync dry run (`DRY_RUN=1`): 504 albums, no errors
|
||||
- [x] Switched 2026-10-24 12:00 UTC: HTTPRoute straight to
|
||||
helm-immich-server (tested beforehand with a temporary route behind a
|
||||
secret header: page, API and a websocket upgrade), auto launch on,
|
||||
password login off, login page message gone, immich restarted to read
|
||||
the config file, the 25 sessions of `foto@` ended (those of
|
||||
`fotoupload@`, phones backing up to the group account, kept),
|
||||
share-sync on, gatus on s1 and s2 checking `/api/server/ping`. The first
|
||||
run hid no invitations: immich stores a notification's `data` as a JSON
|
||||
string inside the jsonb column; fixed, 1008 hidden, the 6 for albums of
|
||||
the last week kept
|
||||
- [x] Removed sol-auth 2026-10-24 (`immich/sol-auth*`,
|
||||
`immich/httpd.conf-cm.yml`). The immich folder app doesn't prune, so its
|
||||
Deployment, Service and ConfigMap went through a sync limited to those
|
||||
three with prune on
|
||||
- [x] `foto@`'s activity (155 likes, no comments) deleted 2026-10-24, and
|
||||
the account marked deleted the way immich's admin delete does (status
|
||||
`deleted`, `deletedAt`); immich removes it for good after 7 days
|
||||
- [x] Nobody uses `fotoupload@` anymore: its 9 (browser) sessions ended
|
||||
2026-10-24. Its API key stays for share-sync
|
||||
|
||||
D. auth.scoutingmbg.nl (2026-10-25)
|
||||
- [x] The group's DNS points scoutingmbg.nl, www, foto and auth at node001
|
||||
(rubenhensen.nl). node001 passes all four through to the home cluster
|
||||
(SNI on 443, a proxy on 80 for HTTP-01), like its other tunneled hosts,
|
||||
and gatus on s1 checks them
|
||||
- [x] Certificate, Gateway listener `https-auth-scoutingmbg`, the authentik
|
||||
route on both hostnames, the brand, and immich's issuer on
|
||||
https://auth.scoutingmbg.nl/application/o/immich/
|
||||
- [x] SOL connection: redirect URI
|
||||
`https://auth.scoutingmbg.nl/source/oauth/callback/scouts-online/`.
|
||||
Editing the connection left the authentik.rubenhensen.nl URI unusable
|
||||
(SOL answers `invalid_client` after the login when the redirect URI isn't
|
||||
registered), so logins failed for a few minutes until immich moved over
|
||||
|
||||
## Open points
|
||||
|
||||
- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
|
||||
scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
|
||||
all the records it moved, and node001 tunnels all four home. ns1 lagged
|
||||
behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
|
||||
uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
|
||||
`a` now means node001, which is fine: the WordPress site sends no mail
|
||||
and the group's mail goes through Google (`mx`)
|
||||
- Public shared links become public for real once sol-auth is gone.
|
||||
- Photos leiding back up from their phones stay private until they add
|
||||
them to a shared album.
|
||||
@@ -0,0 +1,146 @@
|
||||
# A second SSD for Longhorn
|
||||
|
||||
## Why
|
||||
|
||||
Root, the dqlite database, every Longhorn replica, container images and
|
||||
emptyDirs all sit on one Apacer AS350 1TB, a DRAM-less 2.5" SATA SSD. When
|
||||
something writes to it for minutes (Longhorn backups at 00:00, the Velero
|
||||
backup at 01:00, kopia maintenance, image pulls), its write latency climbs
|
||||
and dqlite, which syncs every commit, stalls for seconds. Controllers then
|
||||
lose their leases and restart ("Still stalling after the VACUUM" in
|
||||
docs/upgrade-k8s-1.33.md). Longer leases and throttled maintenance soften
|
||||
that; moving Longhorn's data to its own NVMe SSD takes away most of the
|
||||
writes dqlite competes with.
|
||||
|
||||
The alternative, moving dqlite to the new SSD, is less work in data but
|
||||
not in risk: its directory lives under the microk8s snap's revision
|
||||
directory, which snapd copies on every refresh, so a bind mount there is
|
||||
fragile. Longhorn supports several disks per node and moves replicas
|
||||
between them live.
|
||||
|
||||
## Hardware
|
||||
|
||||
- opti1 is a Dell OptiPlex 5060 (BIOS 1.29.0); the i5-8600T points to the
|
||||
Micro, but check the case. Every 5060 has one M.2 2280 slot that takes
|
||||
PCIe NVMe (Gen3 x4); Dell sells 2280 Gen3 x4 drives for the Micro. The
|
||||
only storage controller is the SATA AHCI one and no NVMe device is
|
||||
present, so the slot should be empty.
|
||||
- Buy an M.2 2280 NVMe SSD **with a DRAM cache**, 2 TB. Gen4 drives work and
|
||||
run at Gen3 speed, which is plenty and cooler. Examples with DRAM:
|
||||
Samsung 990 Pro, WD Black SN850X, Kingston KC3000, Crucial T500. Avoid
|
||||
DRAM-less and QLC drives (WD SN770, Samsung 990 EVO, Crucial P3); they
|
||||
have the same weakness as the current SSD. In the Micro, skip a heatsink
|
||||
that doesn't fit under the cover.
|
||||
- Size: Longhorn holds 339 GiB on disk now, of which 297 GiB is the immich
|
||||
library (126 GB in use inside the volume; the rest is snapshots, see
|
||||
below). Personal immich accounts will add uploads; volumes can be sized
|
||||
up to 551 GiB in total today. 2 TB leaves room; 1 TB would fit for now.
|
||||
|
||||
## Before: shrink what has to move
|
||||
|
||||
- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
||||
snapshots from before October 2026, nearly all on immich-claim
|
||||
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
|
||||
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
|
||||
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
||||
don't depend on them. Deleting a snapshot merges it into the next one,
|
||||
which writes a lot, so do this in a quiet hour, one volume at a time.
|
||||
Happened on 2026-10-24, from a Claude Code session at Ruben's request
|
||||
(the 45 snapshots from before October 2026, small volumes first): immich-claim's
|
||||
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
||||
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
||||
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
||||
average, with "full" IO pressure at 45-75% (`~/iosample.log`); the
|
||||
instance manager wrote 2.3 TB and read 2.8 TB in 30 hours. What is left
|
||||
on immich-claim is one 233 GiB snapshot (`system-backup-0b7f8c6047294b36`,
|
||||
from the Longhorn system backup of 2026-10-22) that everything merged
|
||||
into; 148 snapshots hold 273 GiB across all volumes.
|
||||
- [ ] Delete the snapshots the Longhorn system backups left behind: 116
|
||||
`system-backup-*` snapshots (one per volume for each system backup
|
||||
before an upgrade step on 2026-10-22 and 23) hold 258 GiB. The system
|
||||
backups on the NAS don't need them. The small volumes are cheap; on
|
||||
immich-claim the 233 GiB one is the base the rest merged into, so
|
||||
deleting it means hours of merging like on 2026-10-24: one night, or
|
||||
after the move to the NVMe drive. Then trim (next item); deleted files
|
||||
only free space once both are done. Next time, take system backups with
|
||||
the volume backup policy `disabled` when the nightly backups are fresh,
|
||||
so they create no snapshots
|
||||
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
|
||||
so blocks of deleted files are released.
|
||||
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
|
||||
Longhorn system backup.
|
||||
- [ ] Set Longhorn's `concurrent-replica-rebuild-per-node-limit` from 5 to 1
|
||||
for the move, so only one replica copies at a time. It is not in the
|
||||
Helm values, so set it in the UI (Settings, General) or with
|
||||
`kubectl -n longhorn-system patch settings.longhorn.io concurrent-replica-rebuild-per-node-limit --type merge -p '{"value":"1"}'`.
|
||||
|
||||
## Install (downtime: everything, about 15 minutes)
|
||||
|
||||
- [ ] `sudo shutdown -h now` on opti1. microk8s stops, the volumes detach.
|
||||
- [ ] Unplug, open the case, put the SSD in the M.2 slot and screw it down.
|
||||
- [ ] Boot into the BIOS (F2): the NVMe drive shows up under storage, SATA
|
||||
operation stays AHCI, and the boot sequence still starts "ubuntu" from
|
||||
the SATA SSD.
|
||||
- [ ] Boot. `lsblk` shows `nvme0n1`.
|
||||
- [ ] Unseal Vault, which comes up sealed (until then ESO can't refresh
|
||||
secrets): `for p in vault-0 vault-1 vault-2; do kubectl -n vault exec -ti $p -- vault operator unseal; done`
|
||||
- [ ] Wait until everything is green again (gatus on s1 and s2).
|
||||
|
||||
## Prepare the disk (sudo on opti1)
|
||||
|
||||
```bash
|
||||
lsblk -o NAME,SIZE,MODEL /dev/nvme0n1 # the new, empty drive
|
||||
sudo wipefs -a /dev/nvme0n1
|
||||
sudo parted -s /dev/nvme0n1 mklabel gpt mkpart longhorn ext4 1MiB 100%
|
||||
sudo mkfs.ext4 -L longhorn-nvme /dev/nvme0n1p1
|
||||
sudo mkdir -p /var/lib/longhorn-nvme
|
||||
echo "UUID=$(sudo blkid -s UUID -o value /dev/nvme0n1p1) /var/lib/longhorn-nvme ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
|
||||
sudo systemctl daemon-reload && sudo mount /var/lib/longhorn-nvme
|
||||
df -h /var/lib/longhorn-nvme
|
||||
```
|
||||
|
||||
Longhorn wants an extent-based filesystem (ext4 or xfs) mounted at a
|
||||
directory, no symlink. With `nofail` a missing drive doesn't block the
|
||||
boot; Longhorn then sees an empty directory on the root filesystem with
|
||||
the wrong filesystem ID and won't use it. Ubuntu's weekly `fstrim.timer`
|
||||
covers the new filesystem too.
|
||||
|
||||
## Move Longhorn's data (no downtime)
|
||||
|
||||
- [ ] Add the disk to the Longhorn node:
|
||||
```bash
|
||||
kubectl -n longhorn-system patch nodes.longhorn.io opti1 --type merge -p '{"spec":{"disks":{"nvme":{"path":"/var/lib/longhorn-nvme","diskType":"filesystem","allowScheduling":true,"evictionRequested":false,"storageReserved":53687091200,"tags":[]}}}}'
|
||||
```
|
||||
(50 GiB reserved.) Wait until the disk is Ready and Schedulable.
|
||||
- [ ] On the old disk (`default-disk-5db8685563bf9bf9`, `/var/lib/longhorn/`)
|
||||
set `allowScheduling: false`, then `evictionRequested: true`. Longhorn
|
||||
builds a new replica of each volume on the NVMe drive and drops the old
|
||||
one once the rebuild succeeds; it attaches detached volumes for this
|
||||
itself. All 20 volumes have one replica, so each briefly has two.
|
||||
- [ ] Watch the old disk's replica count go to 0 (UI, Node, or
|
||||
`kubectl -n longhorn-system get nodes.longhorn.io opti1 -o yaml`). The
|
||||
immich library takes longest. Copying reads from the SATA SSD, so expect
|
||||
a few slow API moments meanwhile; best in the evening.
|
||||
- [ ] Remove the old disk from `spec.disks` once it has no replicas, and set
|
||||
the rebuild limit back to 5.
|
||||
|
||||
To stop halfway: set `evictionRequested` back to false; replicas already
|
||||
moved stay on the NVMe drive, the rest on the SATA SSD, and both disks
|
||||
work.
|
||||
|
||||
## Afterwards
|
||||
|
||||
- [ ] Compare with before: lease updates over 5s
|
||||
(`apiserver_request_duration_seconds` for PUT on leases), controller
|
||||
restarts, and sda's latency during the nightly backups
|
||||
(`~/iosample.sh`).
|
||||
- [ ] Optional, if stalls remain: container images
|
||||
(`/var/snap/microk8s/common/var/lib/containerd`) and the kubelet's
|
||||
emptyDirs, such as kopia's scratch space, still write to the SATA SSD.
|
||||
|
||||
## Sources
|
||||
|
||||
- Dell parts for the OptiPlex 5060 Micro, with M.2 2280 PCIe NVMe Gen3 x4
|
||||
drives: https://dell.com/en-au/shop/pfydresults/6163895
|
||||
- Longhorn, eviction: https://longhorn.io/docs/1.13.0/nodes-and-volumes/nodes/disks-or-nodes-eviction/
|
||||
- Longhorn, multiple disks: https://longhorn.io/docs/1.13.0/nodes-and-volumes/nodes/multidisk/
|
||||
+560
@@ -0,0 +1,560 @@
|
||||
# Roadmap
|
||||
|
||||
Where the cluster stands and what is left, as of 2026-10-25. Takes over
|
||||
from wishlist.txt (January 2026, removed); its items are sorted in at the
|
||||
bottom.
|
||||
|
||||
## State
|
||||
|
||||
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
||||
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
||||
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
||||
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
|
||||
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to Garage
|
||||
(MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90 days).
|
||||
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
||||
through ntfy; s2 also gets an hourly report from inside the cluster on the
|
||||
backups, Vault's seal, the Argo CD apps and the CNPG clusters.
|
||||
Every app namespace has NetworkPolicies, and CI renders and validates all
|
||||
apps on every PR. node001 runs mail and tunnels the public hosts home.
|
||||
|
||||
## Follow-ups from this week
|
||||
|
||||
From docs/upgrade-k8s-1.33.md, docs/immich-scouts-online.md and
|
||||
docs/longhorn-second-ssd.md:
|
||||
|
||||
- [x] Read `~/iosample.log` on opti1: the worst stretch was not the
|
||||
backups but 8.5 hours of Longhorn merging immich-claim's deleted
|
||||
snapshots (2026-10-24 14:30-23:15), see docs/upgrade-k8s-1.33.md
|
||||
- [ ] From 2026-10-27, if immich runs without problems: delete
|
||||
`immich/postgresql-cluster.yaml` (the PG14 cluster still runs, 20 GiB),
|
||||
the Longhorn snapshots and backups `immich-pg14-pre-vchord` and
|
||||
`immich-pg17-pre-v3`, and both dumps in `~` on opti1. PostgreSQL 14
|
||||
reaches end of life on 2026-11-12
|
||||
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
||||
(`backend-*.tar`, `dqlite-pre-*`)
|
||||
- [ ] Log in to immich as a jeugdlid (quota 0)
|
||||
- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
|
||||
2026-10-25). The snap is held, so microk8s gets no updates at all until
|
||||
someone refreshes it
|
||||
- [ ] Second SSD, docs/longhorn-second-ssd.md
|
||||
- [x] `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML (PR #39,
|
||||
2026-10-25; the models are cached on a volume)
|
||||
|
||||
## Next, in order
|
||||
|
||||
### 1. hensen.io expires
|
||||
|
||||
foto.scoutingmbg.nl was a CNAME to mbg.hensen.io. Since 2026-10-24 it is a
|
||||
CNAME to rubenhensen.nl (node001, which tunnels it home), but only on the
|
||||
group's ns2: ns1 is stuck on an old serial and still answers
|
||||
mbg.hensen.io (docs/immich-scouts-online.md).
|
||||
|
||||
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
||||
- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
|
||||
`rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
|
||||
cluster uses hensen.io anymore. hensen.io can expire
|
||||
|
||||
### 2. Off-site backups
|
||||
|
||||
Both backup targets are on the same NAS, in the same house as opti1. A
|
||||
fire or burglary takes the cluster, the backups and 45,703 photos that
|
||||
belong to the group.
|
||||
|
||||
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
|
||||
friend.
|
||||
|
||||
- [ ] Ours to his bucket: `rclone sync` from Garage's `velero-backups`
|
||||
(~190 GB) to his bucket through an rclone `crypt` remote, in Docker on
|
||||
the NAS like the copy from MinIO (see 5). Encrypted on our side: the
|
||||
kopia repositories use Velero's default password (`static-passw0rd`),
|
||||
so without it he could read everything, the group's photos included.
|
||||
Keep the crypt passwords next to the Vault unseal key. Daily after the
|
||||
Velero backup, ntfy on failure. Hyper Backup, the earlier plan, backs up
|
||||
shared folders; Garage keeps its objects as blocks plus a metadata
|
||||
database, and a copy of that folder taken while Garage runs is not
|
||||
consistent
|
||||
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
|
||||
now, 500 GB to grow), and versioning or object lock on the bucket, so
|
||||
whoever takes over the NAS can't delete the off-site copy as well
|
||||
- [ ] His backups with us: a bucket and key for him, with a quota, in the
|
||||
Garage on the NAS (`nas/garage`, see 5). Only the S3 API public, as
|
||||
`s3.rubenhensen.nl` through node001 and Envoy with a cert-manager
|
||||
certificate; the admin API stays on the LAN. The cluster side (DNS,
|
||||
node001 tunnel, Certificate, HTTPRoute) goes in git. Needed: how much
|
||||
space he wants, free space on the NAS
|
||||
- [x] Which copy of the immich library goes off-site: Velero's, with the
|
||||
rest of the bucket (decided 2026-10-26). It is most of the ~190 GB, so
|
||||
ask him for room for it
|
||||
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
||||
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
|
||||
Planned on 2026-10-25, not run: a second Velero on the real bucket is
|
||||
risky even with a ReadOnly location, because kopia could then run
|
||||
maintenance on the same repositories from two clusters. Instead: copy
|
||||
one backup (`backups/<name>/`) and the kopia repositories of a few small
|
||||
namespaces (`kopia/{gatus,ntfy,freshrss}/`) read-only from the NAS with
|
||||
rclone into a local folder, serve that with `rclone serve s3`, and
|
||||
restore into a throwaway k3s in Docker (Velero 1.18.2, node agent on,
|
||||
storage class longhorn mapped to local-path, without httproutes,
|
||||
certificates, referencegrants and networkpolicies). It needs the MinIO
|
||||
key outside the cluster, which wants your go-ahead; better still, a
|
||||
MinIO user with read-only access to `velero-backups`.
|
||||
Built as `backup-testing/restore-test.sh` (2026-10-25) and rehearsed
|
||||
end to end against a backup made inside the test cluster, with a
|
||||
ReadOnly location: data restored byte for byte (same sha256), deployment
|
||||
available, everything removed afterwards. Two pitfalls found on the way
|
||||
are in the script's header. Not yet run against the real backups:
|
||||
- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
|
||||
daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
|
||||
volume came back with its backed-up byte count (actual-budget 8.4 MB,
|
||||
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
|
||||
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
|
||||
MinIO runs region `minio`; the script now reads endpoint, region and
|
||||
bucket from Velero's location. Copying freshrss's kopia repository took
|
||||
14 of the ~17 minutes
|
||||
- [x] Ran it again on 2026-10-26 against the copy in Garage
|
||||
(daily-backup-20261026010028, same namespaces), before Velero switched
|
||||
over (see 5): passed, in 5 minutes instead of 17
|
||||
- [ ] Run it on a schedule. It needs Docker and an S3 key, so not in the
|
||||
cluster itself: the Mac (launchd) or node001, ideally with a read-only
|
||||
Garage key (`garage key create`, `garage bucket allow --read`)
|
||||
|
||||
### 3. authentik 2026.2.1 → 2026.8.5
|
||||
|
||||
Since 2026-10-24 authentik is the front door for every member of the
|
||||
group. Renovate PRs #20 (2026.2.3) and #22 (2026.8.5) had been open since
|
||||
April and May.
|
||||
|
||||
- [x] Upgraded 2026-10-25 in two hops, since releases can't be skipped:
|
||||
chart 2026.5.6, then 2026.8.5. Each took about three minutes with no
|
||||
downtime; the old pods kept serving until the new ones were ready.
|
||||
Safety net: `pg_dump -Fc` in `~/authentik-pre-2026.5.dump` (15 MB) on
|
||||
opti1 and Longhorn snapshot `authentik-pg-pre-2026-5`. Checked after
|
||||
each hop: the seven OIDC discovery endpoints and their issuers, the
|
||||
login page, immich, every blueprint successful, the LDAP outpost
|
||||
checking in on the new version
|
||||
- 2026.5 listens on `[::]` instead of `0.0.0.0`. IPv6 is on in the pods
|
||||
(`bindv6only` 0), so IPv4 keeps working
|
||||
- 2026.8 only believes `X-Forwarded-*` from
|
||||
`listen.trusted_proxy_cidrs`, which defaults to the private ranges
|
||||
(10.0.0.0/8 covers Envoy's pods); redirects stay https
|
||||
- The managed LDAP outpost didn't follow on its own after either hop;
|
||||
`outpost_controller.send(<outpost pk>)` in `ak shell` rolled it over
|
||||
- The `redis:` values were left over from before authentik dropped
|
||||
Redis; the chart has no such key, so they went
|
||||
- [x] Log in once through each app: immich through Scouts Online, Argo CD,
|
||||
Vault, Longhorn, FreshRSS, Actual, mail
|
||||
- [ ] After a week without problems: delete the dump and the snapshot
|
||||
- [x] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`), required
|
||||
from 2026.11: already `https://authentik.rubenhensen.nl`, filled in by
|
||||
the 2026.8 migration from the embedded outpost's host. In 2026.8.5 it is
|
||||
only stored; on main it fills itself in the same way and sets the
|
||||
embedded outpost's host from it. OIDC issuers still come from the
|
||||
request's host, so immich's issuer on auth.scoutingmbg.nl is unaffected
|
||||
(checked 2026-10-25 in the source)
|
||||
|
||||
### 4. Vault 1.17.2 → 2.2.1
|
||||
|
||||
Vault unseals with Shamir keys, so after every reboot (the SSD install,
|
||||
for one) it stays sealed and ESO stops refreshing secrets until someone
|
||||
unseals it.
|
||||
|
||||
- [x] Upgraded 2026-10-25, straight from 1.17.2 to 2.2.1. HashiCorp
|
||||
doesn't require going through every release, and the change tracker
|
||||
lists no breaking change that applies here: `disable_mlock` is set
|
||||
(required for raft since 1.20), the chart sets `SKIP_SETCAP` and runs
|
||||
Vault as uid 100 (2.0.1's IPC_LOCK requirement, dropped again in 2.0.2),
|
||||
and ESO uses a token and KV v2. `tools/vault-roll.sh 2.2.1` restarted
|
||||
the standbys first and the active node last, unsealing each; vault-1
|
||||
took over as active within seconds. Safety net: Longhorn snapshots
|
||||
`vault-{0,1,2}-pre-2-2-1`. Afterwards all three ran 2.2.1 at the same
|
||||
raft index, and all 24 ExternalSecrets refreshed after a force-sync
|
||||
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
||||
doesn't exist, so the OIDC mount's listing visibility was never set.
|
||||
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
||||
- [x] There is one unseal key (1 share, threshold 1), kept safe outside
|
||||
this cluster (2026-10-25); without it the data in Vault is lost
|
||||
- [x] Unsealing is in the SSD install steps
|
||||
- [ ] After a week without problems: delete the three snapshots
|
||||
|
||||
### 5. Backups: which tool keeps what
|
||||
|
||||
Findings from 2026-10-25:
|
||||
|
||||
- Velero backs up every mounted volume through kopia, including the immich
|
||||
library (133 GB), which Longhorn also backs up to the same NAS. Most runs
|
||||
take 10s to 3 min, since kopia only reads changed files. 3 of the last 13
|
||||
read everything again: 37 min on 10-21, 30 min on 10-24 (the lease loss
|
||||
at 01:09), 13 min for the weekly on 10-25. Why those runs found no
|
||||
usable parent is unclear; the 10-24 one was the first after the Velero
|
||||
1.18 upgrade restarted the node agent. Once Longhorn's data is on the
|
||||
NVMe SSD these reads no longer hit the disk dqlite is on.
|
||||
- Velero holds the photos longest. Longhorn keeps 7 backups per volume
|
||||
(count-based, so extra runs push out older ones: immich-claim's oldest
|
||||
daily is from 10-22); Velero keeps dailies 30 days and weeklies 90.
|
||||
Dropping immich from Velero means raising Longhorn's retention first.
|
||||
- Longhorn's recurring job skips detached volumes
|
||||
(`allow-recurring-job-while-volume-detached` is false). immich-claim got
|
||||
no Longhorn backup on 10-24: the server was at 0 replicas for the
|
||||
database migration from 22:18 to 00:06 UTC.
|
||||
- Velero copies the CNPG and MariaDB data directories file by file while
|
||||
the databases run, which is not a consistent copy. The Longhorn backups
|
||||
start from a snapshot and are crash-consistent. CNPG can back up to
|
||||
Garage itself (barman cloud plugin), with point-in-time recovery.
|
||||
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
|
||||
maintenance job pods) was fixed that afternoon by
|
||||
`velero/skip-emptydir-policy.yaml`.
|
||||
|
||||
- [x] Whether the immich library stays in Velero: it stays, since
|
||||
Velero's bucket is what goes off-site (2; decided 2026-10-26)
|
||||
- [x] CNPG backups through the barman cloud plugin (PR #56, merged
|
||||
2026-10-26 22:43 UTC): authentik's and immich's PostgreSQL 17 back up
|
||||
to their own bucket in Garage (`cnpg-authentik`, `cnpg-immich`), a base
|
||||
backup at 03:00 UTC and the WAL every 5 minutes, kept 30 days. Adding
|
||||
the sidecar restarted both instances; authentik was down from 22:44 to
|
||||
22:48, since the old pod waited for its connections to close. The first
|
||||
base backups: 46 MB in 29 s and 776 MB in 3.5 minutes. immich's
|
||||
`immediate` backup ran before the plugin was up and failed; a manual one
|
||||
(`immich-postgres-17-first`) went through
|
||||
- [ ] Restore one of them into a new Cluster (`bootstrap.recovery` from
|
||||
the ObjectStore, to a point in time) to see that it works. After that,
|
||||
decide whether Velero still needs to copy the CNPG data directories
|
||||
- [ ] Take `cnpg-authentik` and `cnpg-immich` along in the off-site copy
|
||||
(2)
|
||||
- [x] Velero's target, MinIO on the NAS, is no longer maintained: no
|
||||
community images since October 2025, maintenance mode since December
|
||||
2025, and the repository is archived. Since 2026-10-26 Velero backs up
|
||||
to Garage on the same NAS (`nas/garage`, PR #35). The bucket was copied
|
||||
with rclone (185 GiB in about 6 hours, then a catch-up sync just before
|
||||
the switch), so the existing backups and kopia repositories carried on:
|
||||
after the switch all 55 backups were still listed, and a manual backup
|
||||
of ntfy found the existing repository and used the night's snapshot as
|
||||
its parent. Two things made the NAS slow during the copy: Garage on
|
||||
LMDB with `metadata_fsync`, ~270 disk writes/s while idle (now SQLite,
|
||||
see `nas/garage/garage.toml`), and MinIO's scanner, which kept one disk
|
||||
at 97% busy. The scanner is now at `slowest` (`mc admin config set local
|
||||
scanner speed=slowest` in the MinIO container); it may have slowed
|
||||
Velero's backups to MinIO as well
|
||||
- [ ] Switch MinIO off once Velero has run on Garage for a week (from
|
||||
2026-11-02): stop the container, then delete `/volume1/minio` (343 GiB)
|
||||
and the hand-made Secret `velero/cloud-credentials` (MinIO's key). Only
|
||||
MinIO still has the kopia repositories of namespaces that are gone
|
||||
(~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
|
||||
mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
|
||||
whether any of them is worth keeping
|
||||
- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
|
||||
2026-10-26 next to a 270 MB database, and going by the folders in
|
||||
`meta/snapshots`, each 6-hourly metadata snapshot takes about 70
|
||||
minutes. Not growing; check again after a week of normal use
|
||||
|
||||
### 6. Monitoring
|
||||
|
||||
Nothing collects metrics or logs, not even metrics-server (`kubectl top`
|
||||
fails). The dqlite stalls were tracked down by hand from the API server's
|
||||
metrics, and Velero's scheduled backups failed every night from August
|
||||
until 2026-10-21 without anyone noticing.
|
||||
|
||||
- [x] A stopgap for the worst blind spots (2026-10-25,
|
||||
`gatus/backup-check.yaml`): an hourly CronJob reads the API and pushes
|
||||
six external endpoints to s2 (group "cluster"): the newest daily and
|
||||
weekly Velero backup Completed and recent, every attached Longhorn
|
||||
volume healthy with a backup younger than 26h, no Vault pod sealed,
|
||||
every Argo CD app Synced and Healthy, every CNPG cluster healthy.
|
||||
Gatus alerts through ntfy after two failed reports, or when no report
|
||||
arrives for 3 hours. The token comes from an ESO Password generator.
|
||||
Tested with made-up data for each failure (a Failed or old backup, a
|
||||
degraded or unbacked-up volume, a sealed pod)
|
||||
- [x] metrics-server (2026-10-25, `apps/templates/metrics-server-helm.yaml`):
|
||||
`kubectl top` works; it keeps its data in memory, so it adds no disk
|
||||
writes. The first look: immich v3 checks file checksums every night from
|
||||
03:00 UTC for up to an hour (IntegrityService), reading ~100 MB/s from
|
||||
sda; it doesn't overlap the backups at 00:00 and 01:00
|
||||
- [ ] After the second SSD (a time series database writes all the time
|
||||
and would add to today's stalls): Prometheus or
|
||||
VictoriaMetrics with Grafana, alerting to ntfy. Loki later
|
||||
- [ ] Alerts worth having: disk latency and fill, lease losses and
|
||||
controller restarts, backups that don't complete, Longhorn volumes that
|
||||
aren't healthy, Vault sealed
|
||||
|
||||
### 7. Limits, probes, network policies
|
||||
|
||||
Counted 2026-10-25 over the 73 running containers:
|
||||
|
||||
- 59 have no requests and 64 no memory limit, among them all of Argo CD,
|
||||
authentik, cert-manager, ESO, mariadb-operator and Vault, and freshrss,
|
||||
gitea, immich and serpbear. The node has 32 GB with 14.6 GB in use; one
|
||||
runaway pod can take the node down, dqlite with it.
|
||||
- No probes on gitea, freshrss, serpbear, authentik's LDAP outpost, the
|
||||
ESO controller, dex and Argo CD's redis.
|
||||
- NetworkPolicies exist only in argocd and longhorn-system. Public are
|
||||
immich (every member, plus public share links), mbgwp (WordPress) and
|
||||
authentik.
|
||||
- Running as root as far as the securityContext tells (images not
|
||||
checked): authentik, bolletjestrui, freshrss, gatus, gitea, immich,
|
||||
ntfy, serpbear, mariadb-operator.
|
||||
|
||||
- [x] NetworkPolicies for immich, mbgwp and authentik
|
||||
(`<namespace>/networkpolicy.yaml`, 2026-10-25). Before, WordPress could
|
||||
reach the NAS's DSM, the router, the API server, Vault and immich. Now
|
||||
each of the three reaches only its own pods, DNS and the internet
|
||||
(without the private ranges), plus the API server for the CNPG
|
||||
instances and authentik's worker. Ingress only from Envoy, the CNPG and
|
||||
MariaDB operators, the node (kubelet probes) and, for the LDAP outpost,
|
||||
from anywhere. Checked afterwards: the sites, the OIDC endpoints and
|
||||
logins, Longhorn's login through Envoy, the LDAP NodePort, CNPG
|
||||
healthy, a MariaDB backup and a share-sync run. The node's addresses
|
||||
(192.168.1.204 and Calico's 10.1.7.64) are written out, so a new node
|
||||
means new policies
|
||||
- [x] The same for gitea, freshrss, serpbear, ntfy, actual-budget,
|
||||
bolletjestrui and gatus (2026-10-25), plus gatus → ntfy in-cluster.
|
||||
home-assistant has no pods (it points at 192.168.1.202). Vault now only
|
||||
takes connections from ESO, Envoy (the UI) and the node (the injector's
|
||||
webhook); its pods reach each other, DNS, the API server and the
|
||||
internet. Checked: every host answers as before, gatus s2 16/16, the
|
||||
OIDC logins of Vault and FreshRSS reach authentik, all 24
|
||||
ExternalSecrets refresh, raft intact; Gitea no longer reaches the NAS
|
||||
or Vault, and a pod in `default` no longer reaches Vault or Gitea
|
||||
- [ ] Certificates couldn't renew in those 11 namespaces: Envoy may only
|
||||
reach each app's own port, and cert-manager's HTTP-01 solver listens on
|
||||
8089. ynab's renewal hung from 2026-10-26 17:35 UTC; a policy for the
|
||||
solver pods, applied by hand in actual-budget that evening, let it
|
||||
through within 20 seconds (now valid until 2027-01-24). PR #51 adds it
|
||||
to all 11. Without it the next renewals hang too: gitea on 11-15,
|
||||
bolletjestrui on 11-18, the rest from 12-03
|
||||
- [x] The infrastructure namespaces (cert-manager, cnpg-system,
|
||||
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
||||
default), merged 2026-10-26: PR #54 for envoy-gateway-system (22:31
|
||||
UTC), PR #53 for the rest (22:42 UTC; one commit per namespace, and
|
||||
`default` denies everything). Checked afterwards: all 16 public hosts
|
||||
answer as before; server-side dry runs reach the CNPG, ESO,
|
||||
cert-manager and MariaDB webhooks (all fail closed); Velero's location
|
||||
Available, every ExternalSecret synced, the dns-sync job ran, no errors
|
||||
from the operators after the barman plugin had started. Velero's policy
|
||||
only lets it reach Garage, not MinIO's port 9000
|
||||
- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
|
||||
had none except the Vault server, from `kubectl top` every 5 minutes
|
||||
from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
|
||||
floor). Rolled out in about eight minutes; both CNPG instances
|
||||
restarted and came back healthy, the OIDC endpoints, the logins and
|
||||
immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
|
||||
1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
|
||||
requested. Still without requests: Longhorn (22 containers, its chart),
|
||||
the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
|
||||
kube-system
|
||||
- [x] Memory limits: PR #58, for everything that got requests in #33.
|
||||
About twice the highest memory from `kubectl top` every 5 minutes from
|
||||
2026-10-26 22:29 to 2026-10-27 21:55 UTC or the night of #33, rounded
|
||||
up; no CPU limits. No OOM kills that day. immich's machine learning
|
||||
sat idle all day (240Mi), so on 2026-10-28 a test image went through
|
||||
CLIP, face recognition and OCR inside the container: 884Mi with all
|
||||
three loaded, limit 2Gi
|
||||
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
|
||||
2026-10-25, its third restart: a crawler asked for 88 pages and assets
|
||||
within a few seconds (normally ~6 a minute), Apache started a PHP worker
|
||||
for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
|
||||
the container went past its 1Gi limit. Unrelated to the
|
||||
NetworkPolicies. Now capped at 12 workers through the chart's
|
||||
`apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
|
||||
with `apache2ctl -t` in the same image first. Replaying the burst (90
|
||||
page requests, 40 at a time): all 200, at most 13 Apache processes and
|
||||
572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
|
||||
peak) instead of a crash
|
||||
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
|
||||
cluster between Envoy and WordPress that keeps anonymous GETs for a few
|
||||
minutes and passes anything with a WordPress cookie through. A page
|
||||
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
|
||||
waiting in a burst is queueing; the cap alone already prevents the crash
|
||||
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
|
||||
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
|
||||
little here, `wordpress-importer` is usually for a one-off import, and
|
||||
`code-snippets` runs PHP from the dashboard
|
||||
- [ ] Rate limiting per visitor in Envoy, on hold. Since PR #50
|
||||
(2026-10-26) node001 passes the visitor's address on with the PROXY
|
||||
protocol, and the apps see it: authentik logs it, Apache logs it for
|
||||
WordPress, so Wordfence's lockout after 20 failed logins now hits one
|
||||
visitor instead of everyone. Envoy Gateway's local rate limit can't
|
||||
count per address (no `Distinct`); that takes its global rate limit,
|
||||
with a Redis and a ratelimit service next to it. Not worth two more
|
||||
components on the one node while the apps' own limits work
|
||||
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
|
||||
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
|
||||
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
|
||||
SSD's stalls. Still without probes: authentik's LDAP outpost (deployed
|
||||
by authentik), the ESO controller, dex and Argo CD's redis (upstream
|
||||
manifests)
|
||||
- [ ] Non-root where the image allows it
|
||||
|
||||
### 8. Renovate and CI
|
||||
|
||||
On 2026-10-25 10 Renovate PRs were open, the oldest from February, and
|
||||
nothing checked a PR before Argo CD applied it. All of them have since been
|
||||
merged or closed (below), none are open (checked 2026-10-25), and CI checks
|
||||
every PR.
|
||||
|
||||
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
|
||||
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
|
||||
does (the apps chart, every Application's chart with its values, value
|
||||
files and parameters, kustomizations, plain folders, and
|
||||
external-snapshotter cloned with its patches) and validates the result
|
||||
with kubeconform (strict, CRD schemas from the datree catalog), plus a
|
||||
duplicate-key check on every YAML file. Helm 4.2.1 like Argo CD 3.5,
|
||||
versions pinned and checksummed, kept current by Renovate through the
|
||||
`# renovate:` comments. Tested by planting a typo in a Deployment, a
|
||||
chart version that doesn't exist and a value the immich chart's schema
|
||||
rejects; all three fail the run. Locally:
|
||||
`docker run --rm -v $PWD:/repo:ro <image with helm, kustomize, kubeconform, PyYAML> python /repo/tools/ci/validate.py`
|
||||
- It found `syncPolicy.automated` twice in
|
||||
`apps/templates/snapshot-controller-helm.yaml` (once with `prune: true`,
|
||||
added 2025-03-03); Argo CD kept the last one, without prune. Now listed
|
||||
once, as it ran. Prune was probably meant; turn it on deliberately if so
|
||||
- [x] This repo's own rules, in `tools/ci/validate.py` rather than
|
||||
conftest (2026-10-25), on what the repo's own folders render: no Secret
|
||||
with data in git, and every image has a tag other than latest (both
|
||||
fail the run; tested with a planted Secret and an untagged image), plus
|
||||
PR annotations for containers without requests or probes. freshrss's
|
||||
`alpine/git` init container had no tag; now `v2.52.0`
|
||||
- [ ] Automerge patch versions of low-risk apps now that CI runs: PR #52
|
||||
(ntfy, gatus, freshrss, serpbear, actual-budget and CI's own tools;
|
||||
three days after the release, only on weekday evenings and at the
|
||||
weekend, when the cluster can be reached)
|
||||
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
|
||||
`argocd` manager and a custom `github-tags` regex both matched. The
|
||||
regex is gone; Renovate should close #15 by itself
|
||||
- [x] Renovate only looked at six folders for images and only at
|
||||
apps/templates for Applications, so it never offered updates for the
|
||||
vault, immich, velero, mariadb-operator and actualbudget charts, or for
|
||||
images in freshrss, mbgwp, dns, vault and immich, or for CNPG's
|
||||
`imageName` (authentik still runs PostgreSQL 17.4). Both managers now
|
||||
cover every app folder, a regex manager reads `imageName` in
|
||||
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
|
||||
(those need pg_upgrade or a dump). Expect a batch of new PRs
|
||||
(2026-10-25). They only come once fewer than 10 Renovate PRs are open
|
||||
(`config:recommended`'s `prConcurrentLimit`); until then they wait
|
||||
under "Rate-Limited" on the Dependency Dashboard (#2)
|
||||
- The other folders were left out with a `(?!…)` lookahead. Renovate
|
||||
compiles these patterns with RE2, which has no lookahead, so it
|
||||
rejected the config and stopped opening PRs (#32). Each manager now
|
||||
takes every folder and skips the others through its own
|
||||
`ignorePaths`. Renovate 44.181.6 with RE2 loaded fails the old config
|
||||
with #32's error and passes the new one, and its matcher picks the
|
||||
same 166 files as the old pattern did
|
||||
- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
|
||||
v1.21.3 (security release; all three pods on v1.21.3), #24
|
||||
external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
|
||||
v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
|
||||
0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
|
||||
ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
|
||||
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
|
||||
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
|
||||
its own beforehand, the nightly ones are the way back)
|
||||
- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
|
||||
manual rebase (#33 had added values on the next line, which git counts
|
||||
as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
|
||||
ExternalSecrets refreshed on a force-sync; the 25th is the generated
|
||||
`gatus/backup-check-token`, which never refreshes by design
|
||||
|
||||
### 9. One node
|
||||
|
||||
One node, one SSD, one replica per volume. The second SSD takes away the
|
||||
disk contention; a second and third node (nix-infra, like node001) is the
|
||||
step after that, if downtime starts to matter.
|
||||
|
||||
### 10. Repo and cluster cleanup
|
||||
|
||||
- [x] Done 2026-10-25: `disable/` and the `.DS_Store` files, now in
|
||||
`.gitignore`; `secrets/hensenio-smtp.yaml` and `velero/minio-es.yaml`
|
||||
(commented out in full), and with the first the `secrets-folder` app;
|
||||
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
||||
completed kopia maintenance jobs of repositories that no longer exist
|
||||
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
||||
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
|
||||
volumes over the file paths `data/config.custom.php` and
|
||||
`data/config-user.custom.php`, so FreshRSS sees two empty directories
|
||||
there and the volumes hold nothing (0 bytes in every backup). Drop both
|
||||
mounts and the two PVCs, or make them ConfigMap files if custom config
|
||||
is wanted
|
||||
- [x] The namespace `secrets` was already gone on 2026-10-26
|
||||
- [x] The Failed Velero backups (the dailies from 2026-09-26 to 10-21 and
|
||||
the weeklies since 08-02; 40 by then, one had expired), deleted through
|
||||
Velero on 2026-10-26. Left: 13 Completed, 3 PartiallyFailed
|
||||
- [ ] The mariadb-operator Backup `mbgwp-pre-migration-backup`, a dump
|
||||
from 2026-02-22 on its own 10 GiB volume (which has a Longhorn backup
|
||||
from 2026-10-22). The PVC has no owner, so it goes separately:
|
||||
`kubectl -n mbgwp delete backups.k8s.mariadb.com mbgwp-pre-migration-backup && kubectl -n mbgwp delete pvc mbgwp-pre-migration-backup`
|
||||
- [ ] 50 of the 70 Longhorn backup volumes on the NAS belong to volumes
|
||||
that no longer exist. Keep the final backups of 2026-10-22 (ghost-blog,
|
||||
stalwart, whisper). The rest are from apps removed long ago: nextcloud
|
||||
(2025-03), owntracks, traccar, ttrss, whisper-priv, whisper-webui,
|
||||
ghost, sogo, sogo-mail, outrun-game, the old mbg-wordpress and
|
||||
mbgwp-helm volumes, immich's database before CNPG. Decide per app and
|
||||
delete in the Longhorn UI (Backup)
|
||||
- [ ] opti1, without sudo: `rm ~/calico-3.29.3.yaml ~/cni-3.25.1-backup.yaml ~/hosts.toml.docker.io.bak ~/dqlite-vacuum`,
|
||||
rollback material for steps that can't be undone any more (the tool's
|
||||
source is in tools/)
|
||||
- [ ] opti1, with sudo: `/root/backend-*` and `/root/dqlite-pre-*` (from
|
||||
before 1.33, 1.34 and the VACUUM; 1.34 migrated the schema, so none of
|
||||
them fits 1.36) and `~/containerd.bak` (root-owned, from 2025-02)
|
||||
- [ ] `rm -r ~/cadir` on opti1: a CA key and certificate made four minutes
|
||||
before the real keyUsage fix on 2026-03-21 (OpenSSL's default subject
|
||||
"Internet Widgits Pty Ltd", valid for a year) and never installed;
|
||||
microk8s's CA is the `CN=microk8s-ca` one from 19:16 that day, valid
|
||||
until 2036
|
||||
|
||||
### 11. Known vulnerabilities
|
||||
|
||||
A one-off scan on 2026-10-25 of all 52 images running in the cluster
|
||||
(Trivy 0.75.0, remote, linux/amd64, HIGH and CRITICAL only). Counts are
|
||||
critical / high:
|
||||
|
||||
| Image | Crit | High | What to do |
|
||||
|---|---|---|---|
|
||||
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; goes with the cleanup from 2026-10-27 |
|
||||
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. Renovate now offers 17.7; a `17.x-standard-trixie` image is more current |
|
||||
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
||||
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
||||
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
||||
| gitea/gitea:1.27.3 | 8 | 43 | now 28.2.0 (PR #26, 2026-10-25); not scanned again |
|
||||
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
|
||||
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
|
||||
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
|
||||
| coredns/coredns:1.10.1 | 5 | 70 | see below |
|
||||
| dexidp/dex:v2.45.1, argocd v3.5.6, redis 8.2.3 | 4, 1, 2 | | come with Argo CD |
|
||||
| mariadb:11.4.5 | 4 | 65 | the newest 11.4 patch; Renovate now covers it |
|
||||
| calico v3.29.3 | 2-3 | ~50 | comes with microk8s |
|
||||
| goauthentik ldap / server 2026.8.5 | 3, 1 | 74, 65 | the next authentik patch |
|
||||
|
||||
Nothing critical in Envoy, Vault, Velero, cert-manager, ESO, ntfy, gatus or
|
||||
metrics-server.
|
||||
|
||||
- [ ] 16 volumes still have their engine in Longhorn 1.7.3's instance
|
||||
manager (`instance-manager-07d66a5…`, running since March): vault ×3,
|
||||
authentik-postgres, immich-postgres (PG14), mbgwp (mariadb and
|
||||
wordpress), gitea, freshrss ×3, ntfy, gatus, serpbear, bolletjestrui,
|
||||
actual-budget. On one node a pod restart never detaches the volume (the
|
||||
new pod lands on the same node), so only a real detach moves the engine
|
||||
to 1.13's instance manager: scale to 0, wait until Longhorn shows the
|
||||
volume detached, scale back (Argo CD's selfHeal undoes a scale-down, so
|
||||
pause it for the app). Vault comes back sealed; CNPG clusters want
|
||||
`cnpg.io/hibernation=on` instead of a scale-down. The old instance
|
||||
manager goes away by itself once it serves nothing
|
||||
- [ ] CoreDNS runs 1.10.1 from when the addon was enabled; the addon
|
||||
repository has 1.14.3. `sudo microk8s disable dns && sudo microk8s enable dns:1.1.1.1`
|
||||
(a short DNS outage for the whole cluster; check the Corefile after)
|
||||
- [ ] The Trivy Operator for scans that keep running (on the wishlist)
|
||||
|
||||
## wishlist.txt (2026-01-25)
|
||||
|
||||
- Done: Renovate; secrets out of git (Vault and ESO); SSO through authentik
|
||||
(Argo CD, Vault, Longhorn, FreshRSS, Actual, mail, immich); Gitea as a
|
||||
GitHub alternative; nix-infra for node001; most of the repo cleanup.
|
||||
- Partly: network policies, probes, requests and limits, non-root (7);
|
||||
backup testing (a script, not scheduled; 2); understanding Longhorn
|
||||
(docs/longhorn-second-ssd.md).
|
||||
- Open: Prometheus, Grafana, Loki, Alertmanager and which alerts (6);
|
||||
conftest and CI/CD (8); Trivy; a staging environment; an image registry;
|
||||
three nodes and opti1 on nix-infra (9); less manual work per new app.
|
||||
kube-hunter is no longer developed; the Trivy Operator or kube-bench
|
||||
cover that ground.
|
||||
- Dropped: shrinking the immich PVC to 200 GB. Longhorn can't shrink a
|
||||
volume, and with personal accounts the library will grow. Deleting stale
|
||||
snapshots (docs/longhorn-second-ssd.md) frees more.
|
||||
@@ -138,6 +138,13 @@ snapshots after every step. Fresh Velero backup before starting.
|
||||
dropped mbg.hensen.io; that reissue went through the Gateway's HTTP-01
|
||||
solver. The Ingresses were pruned with `PrunePropagationPolicy=orphan`
|
||||
so the Certificates stayed. Gatus now expects a 302 from Longhorn
|
||||
- [x] Retries (`gateway/retry-policy.yaml`, 2026-10-24). When a backend
|
||||
closes a kept-alive connection just as Envoy sends a request over it,
|
||||
Envoy answers 503 (flag UC) and, unlike nginx, does not retry. Behind
|
||||
sol-auth's Apache, which recycled its processes after 30 connections,
|
||||
that cost immich chunks during page loads ("error loading dynamically
|
||||
imported module"). A BackendTrafficPolicy on the Gateway retries twice
|
||||
on connect failures and resets
|
||||
|
||||
## 9. immich: PostgreSQL 14 with pgvecto.rs → 17 with VectorChord
|
||||
|
||||
@@ -166,7 +173,24 @@ pgvecto.rs. Done 2026-10-24, about an hour of immich downtime:
|
||||
(`immich/postgresql-cluster.yaml`), the snapshot/backup and the dump.
|
||||
Rolling back means reverting the old volume to the snapshot, since its
|
||||
embeddings are now `real[]` without pgvecto.rs
|
||||
- [ ] immich 3.x, separately (pgvecto.rs no longer matters for it)
|
||||
- [x] immich 3.x: v2.7.5 → v3.3.5 with chart 0.10.3 → 0.13.4 on
|
||||
2026-10-24, about three minutes of downtime (two of them pulling the
|
||||
image). Charts after 0.12.0 are only in the OCI registry
|
||||
(`ghcr.io/immich-app/immich-charts`), and since 0.13.0 the values
|
||||
schema rejects size, accessMode and storageClass on emptyDir volumes.
|
||||
None of the env variables v3 removed were set; the ML image needs
|
||||
x86-64-v2, and the i5-8600T is v3. The migrations took 16s, both vector
|
||||
indexes stayed, and `immich-admin schema-check` reports no drift (the
|
||||
geodata_places_pkey warning at startup came from the geodata import
|
||||
running at the same time). Safety net, since immich can't be
|
||||
downgraded: Longhorn snapshot `immich-pg17-pre-v3` and
|
||||
`~/immich-pg17-pre-v3.dump` (440 MB) on opti1
|
||||
- [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` on the ML container
|
||||
(v3.3.0), for faster ML with less memory; numerically the same models,
|
||||
so no jobs to rerun. The model cache is an emptyDir, so the one-off
|
||||
preparation repeats on every restart
|
||||
- [ ] After a few days without problems: delete the `immich-pg17-pre-v3`
|
||||
snapshot and dump together with the PG14 leftovers above
|
||||
|
||||
## k8s-dqlite write stalls (found 2026-10-22)
|
||||
|
||||
@@ -212,6 +236,68 @@ do the first three steps: `ssh -t ruben@192.168.1.204 ./dqlite-tuning.sh`
|
||||
`--cgroups-per-qos=false`, the likely trigger of the 1.32->1.33 kubelite
|
||||
panic (#5057), is not set in `args/kubelet`.
|
||||
|
||||
### Still stalling after the VACUUM (2026-10-24)
|
||||
|
||||
The smaller database fixed the snapshot stalls, but controllers kept
|
||||
losing their leases a few times a day: since the API server started,
|
||||
256 of ~278k lease updates took over 5s and 44 over 10s. dqlite's log
|
||||
shows short complete stops (at 10:00:42 even its own health query hit
|
||||
its deadline). Every lease loss that day lined up with minutes of heavy
|
||||
writes to the one SSD, which holds root, dqlite, every Longhorn replica,
|
||||
image layers and emptyDirs:
|
||||
|
||||
| Lease losses | Writing at the time |
|
||||
|---|---|
|
||||
| 00:02 | Longhorn backups of every volume (from 00:00) |
|
||||
| 01:09 | Velero backup (kopia reading and caching 133 GB of photos) |
|
||||
| 09:22 | full kopia maintenance of mbgwp (6 min) and argocd (11 min) |
|
||||
| 10:00:43 | full kopia maintenance of vault (9 min) |
|
||||
| 10:31, 10:38 | immich upgrade: image pulls, pg_dump, migrations |
|
||||
|
||||
Velero ran maintenance every hour for each of the 15 kopia repositories
|
||||
(15 pods an hour), without resource limits and with the kopia cache in
|
||||
an emptyDir; about once a day a run is a full maintenance of minutes, at
|
||||
whatever time of day the hourly schedule had drifted to.
|
||||
|
||||
- [x] Velero: `defaultRepoMaintainFrequency: 24h`, the 15 existing
|
||||
BackupRepositories patched to 24h by hand, and maintenance jobs capped
|
||||
at 500m CPU (2Gi memory)
|
||||
- [x] Leases of 60s/40s/5s instead of 15s/10s/2s (duration, renew deadline,
|
||||
retry period) for the snapshot-controller (kustomize patch in its Argo
|
||||
CD app), the CNPG operator (`additionalArgs`) and Envoy Gateway
|
||||
(`provider.kubernetes.leaderElection`). Longhorn's CSI sidecars and
|
||||
global manager have no setting for it
|
||||
- [x] `~/iosample.sh` on opti1 logs sda's counters and io pressure every 2s
|
||||
for 26h (from 2026-10-24 12:35 UTC) into `~/iosample.log`, to put
|
||||
numbers on the disk latency during tonight's backups. Read on 2026-10-25
|
||||
at 02:35 UTC, per 10 minutes:
|
||||
- Quiet: ~50 writes/s, 0.5-2 ms per write on average, "full" IO
|
||||
pressure under 15%
|
||||
- 14:30-23:15: 250-440 writes/s and 200-400 reads/s, 6-20 ms per write,
|
||||
pressure 45-75%. Longhorn merging immich-claim's deleted snapshots
|
||||
(docs/longhorn-second-ssd.md)
|
||||
- Longhorn backups 00:00-00:15: up to 1,220 reads/s, pressure 69%; the
|
||||
Velero backup at 01:00 was incremental and short (pressure 29%)
|
||||
- The authentik and Vault upgrades 01:20-01:40 (image pulls,
|
||||
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
||||
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
||||
their leases at 01:31
|
||||
Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
|
||||
11:30, leaving out the CNPG range cancels): all 67 minutes with errors
|
||||
had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
|
||||
"full" pressure at 45-78%, against a median minute peak of 24 ms. The
|
||||
worst minute after the merges was the rollout of eight merged PRs at
|
||||
2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
|
||||
CNPG and Envoy Gateway held their 60s leases through all of it
|
||||
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
||||
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
||||
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
||||
- [ ] Structural fix: a second SSD, see docs/longhorn-second-ssd.md. The OptiPlex 5060 (Micro, going by the
|
||||
i5-8600T) has an M.2 slot that takes NVMe, and no NVMe device is
|
||||
present; the Apacer AS350 is a DRAM-less 2.5" SATA SSD. An NVMe SSD with
|
||||
DRAM for Longhorn's data (replicas move over live by evicting the old
|
||||
disk) would leave the SATA SSD to dqlite and the system
|
||||
|
||||
## On to 1.36
|
||||
|
||||
microk8s goes up one minor per refresh. 1.34 brings k8s-dqlite 1.8.x with
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# External Secrets' three pods reach each other, DNS and the API server
|
||||
# (the node, port 16443); the controller also reads Vault
|
||||
# (vault-active.vault.svc:8200, which vault/networkpolicy.yaml lets it
|
||||
# into). Besides their own namespace they take connections only from the
|
||||
# node: the API server calls the webhook from there and the kubelet probes
|
||||
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||
# projectcalico.org/IPv4VXLANTunnelAddr). The Password generator behind
|
||||
# gatus/backup-check-token needs no network. The chart's Application
|
||||
# (apps/templates/external-secrets-helm.yaml) only installs the chart, so
|
||||
# this folder gets an Application of its own, external-secrets-folder.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: controller-to-vault
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-secrets
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: vault
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vault
|
||||
ports:
|
||||
- port: 8200
|
||||
@@ -23,7 +23,7 @@ spec:
|
||||
io.kompose.service: freshrss
|
||||
spec:
|
||||
initContainers:
|
||||
- image: alpine/git
|
||||
- image: alpine/git:v2.54.0
|
||||
name: git
|
||||
command:
|
||||
- git
|
||||
@@ -74,11 +74,30 @@ spec:
|
||||
value: Europe/Amsterdam
|
||||
# 1.30.3 ships PHP 8.4.27, which fixes CVE-2026-109618 and
|
||||
# CVE-2026-109619.
|
||||
image: freshrss/freshrss:1.30.3
|
||||
image: freshrss/freshrss:1.30.4
|
||||
name: freshrss
|
||||
ports:
|
||||
- containerPort: 80
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 176Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
|
||||
# browser.
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 80
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 80
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
volumeMounts:
|
||||
- mountPath: /var/www/FreshRSS/data
|
||||
name: data
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# FreshRSS faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces.
|
||||
# Feeds and the OIDC login (authentik's public address) go out over
|
||||
# the internet.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 80
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -58,6 +58,20 @@ spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: authentik
|
||||
- name: https-auth-scoutingmbg
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
hostname: auth.scoutingmbg.nl
|
||||
tls:
|
||||
certificateRefs:
|
||||
- name: auth-scoutingmbg-tls
|
||||
namespace: authentik
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Selector
|
||||
selector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: authentik
|
||||
- name: https-vic
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# Envoy Gateway: the controller and the proxy it runs (envoyproxy.yaml).
|
||||
# Both reach each other (the proxy gets its config from the controller on
|
||||
# 18000) and DNS, and take connections from each other and the node: the
|
||||
# API server calls the controller's topology webhook (9443) from there and
|
||||
# the kubelet probes come from there too (10.1.7.64 is the node's Calico
|
||||
# VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr). Nothing scrapes
|
||||
# the metrics on 19001.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
namespace: envoy-gateway-system
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# The proxy holds the node's ports 80 and 443 (10080 and 10443 inside) and
|
||||
# takes connections there from anywhere: the internet through node001 and
|
||||
# the router, the LAN, and pods that call a public hostname. It reaches
|
||||
# every app it routes to (pods in any namespace; their own policies decide
|
||||
# which port), Home Assistant on 192.168.1.202:8123 (outside the cluster,
|
||||
# home-assistant/), and the internet on 443 for Longhorn's OIDC login
|
||||
# (authentik's token endpoint, at its public address).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: proxy
|
||||
namespace: envoy-gateway-system
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
app.kubernetes.io/component: proxy
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- ports:
|
||||
- port: 10080
|
||||
- port: 10443
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector: {}
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.202/32
|
||||
ports:
|
||||
- port: 8123
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
ports:
|
||||
- port: 443
|
||||
---
|
||||
# Everything but the proxy (the controller, and the chart's certgen job on
|
||||
# upgrades) reaches the API server (the node, port 16443). The controller
|
||||
# also fetches the OIDC discovery document of Longhorn's SecurityPolicy
|
||||
# from authentik's public address.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: control-plane
|
||||
namespace: envoy-gateway-system
|
||||
spec:
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: app.kubernetes.io/name
|
||||
operator: NotIn
|
||||
values:
|
||||
- envoy
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
ports:
|
||||
- port: 443
|
||||
@@ -0,0 +1,23 @@
|
||||
# node001 passes TLS for the tunneled hosts through without opening it, so
|
||||
# without this every request arrives from 46.224.26.65. Wordfence locks that
|
||||
# address out after 20 failed logins (2026-10-26), and with it everyone who
|
||||
# logs in to scoutingmbg.nl. node001 puts a PROXY header with the visitor's
|
||||
# address in front of each connection; Envoy takes the address from there
|
||||
# and adds it to X-Forwarded-For.
|
||||
#
|
||||
# Optional, because s2 and ntfy2 point straight at home and their clients
|
||||
# send no header, and neither does node001's proxy on port 80. The price:
|
||||
# whoever connects to the home address directly can send a header of their
|
||||
# own and claim any address.
|
||||
apiVersion: gateway.envoyproxy.io/v1alpha1
|
||||
kind: ClientTrafficPolicy
|
||||
metadata:
|
||||
name: proxy-protocol
|
||||
namespace: envoy-gateway-system
|
||||
spec:
|
||||
targetRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: homelab
|
||||
proxyProtocol:
|
||||
optional: true
|
||||
@@ -0,0 +1,25 @@
|
||||
# Retry a request when the backend drops a kept-alive connection just as
|
||||
# Envoy sends a request over it. Envoy answers that with a 503 (flag UC) and
|
||||
# does not retry by default; ingress-nginx did. It broke immich's page loads
|
||||
# behind the Apache of sol-auth (since removed), which recycled its
|
||||
# processes after 30 connections, and shows up less often elsewhere.
|
||||
apiVersion: gateway.envoyproxy.io/v1alpha1
|
||||
kind: BackendTrafficPolicy
|
||||
metadata:
|
||||
name: retry-on-reset
|
||||
namespace: envoy-gateway-system
|
||||
spec:
|
||||
targetRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: homelab
|
||||
retry:
|
||||
numRetries: 2
|
||||
retryOn:
|
||||
triggers:
|
||||
- connect-failure
|
||||
- reset
|
||||
perRetry:
|
||||
backOff:
|
||||
baseInterval: 25ms
|
||||
maxInterval: 250ms
|
||||
@@ -0,0 +1,285 @@
|
||||
# Checks what Gatus can't see from outside and reports it to Gatus as
|
||||
# external endpoints (group "cluster" on s2), which alert through ntfy when
|
||||
# a check fails or when no report arrives (heartbeat). Velero's scheduled
|
||||
# backups failed every night from August to 2026-10-21 without anyone
|
||||
# noticing; a sealed Vault after a reboot looks fine from outside too.
|
||||
#
|
||||
# Every hour, from the Kubernetes API:
|
||||
# - velero-daily / velero-weekly: the newest finished backup of the
|
||||
# schedule is Completed and younger than 26h / 8 days
|
||||
# - longhorn: every attached volume is healthy and has a completed backup
|
||||
# younger than 26h (detached volumes are skipped; the recurring job skips
|
||||
# them too)
|
||||
# - vault-unsealed: all Vault pods report vault-sealed=false
|
||||
# - argocd-apps: every Argo CD application is Synced and Healthy
|
||||
# - postgres: every CloudNativePG cluster is in a healthy state
|
||||
#
|
||||
# The token is generated by ESO and never leaves the cluster; Gatus reads
|
||||
# it from the same Secret.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: backup-check-token
|
||||
spec:
|
||||
length: 40
|
||||
digits: 10
|
||||
symbols: 0
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
secretKeys:
|
||||
- BACKUP_CHECK_TOKEN
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: backup-check-token
|
||||
spec:
|
||||
# Generate once; a new token would need a Gatus restart.
|
||||
refreshInterval: "0"
|
||||
target:
|
||||
name: backup-check-token
|
||||
creationPolicy: Owner
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: backup-check-token
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: backup-check
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: backup-check
|
||||
rules:
|
||||
- apiGroups: ["velero.io"]
|
||||
resources: ["backups"]
|
||||
verbs: ["list"]
|
||||
- apiGroups: ["longhorn.io"]
|
||||
resources: ["backups", "volumes"]
|
||||
verbs: ["list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["list"]
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["applications"]
|
||||
verbs: ["list"]
|
||||
- apiGroups: ["postgresql.cnpg.io"]
|
||||
resources: ["clusters"]
|
||||
verbs: ["list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: backup-check
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: backup-check
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: backup-check
|
||||
namespace: gatus
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: backup-check
|
||||
data:
|
||||
check.py: |
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
SA = "/var/run/secrets/kubernetes.io/serviceaccount"
|
||||
API = "https://kubernetes.default.svc"
|
||||
GATUS = os.environ.get("GATUS_URL", "http://gatus:8080")
|
||||
TOKEN = os.environ["BACKUP_CHECK_TOKEN"]
|
||||
NOW = dt.datetime.now(dt.timezone.utc)
|
||||
FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"}
|
||||
|
||||
with open(f"{SA}/token") as f:
|
||||
sa_token = f.read()
|
||||
api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt")
|
||||
|
||||
|
||||
def items(path):
|
||||
req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"})
|
||||
with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r:
|
||||
return json.load(r)["items"]
|
||||
|
||||
|
||||
def parse(ts):
|
||||
return dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def age(ts):
|
||||
hours = (NOW - parse(ts)).total_seconds() / 3600
|
||||
return f"{hours:.0f}h"
|
||||
|
||||
|
||||
def velero(schedule, max_hours):
|
||||
backups = [
|
||||
b for b in items("/apis/velero.io/v1/namespaces/velero/backups")
|
||||
if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule
|
||||
and b.get("status", {}).get("phase") in FINISHED
|
||||
]
|
||||
if not backups:
|
||||
return f"no finished {schedule} backup"
|
||||
newest = max(backups, key=lambda b: b["status"].get("startTimestamp", ""))
|
||||
status = newest["status"]
|
||||
name = newest["metadata"]["name"]
|
||||
if status["phase"] != "Completed":
|
||||
return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)"
|
||||
done = status.get("completionTimestamp")
|
||||
if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600:
|
||||
return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old"
|
||||
return None
|
||||
|
||||
|
||||
def longhorn(max_hours):
|
||||
newest = {}
|
||||
for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"):
|
||||
s = b.get("status", {})
|
||||
if s.get("state") == "Completed" and s.get("backupCreatedAt"):
|
||||
v = s.get("volumeName")
|
||||
newest[v] = max(newest.get(v, ""), s["backupCreatedAt"])
|
||||
problems = []
|
||||
for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"):
|
||||
s = v.get("status", {})
|
||||
if s.get("state") != "attached":
|
||||
continue
|
||||
name = v["metadata"]["name"]
|
||||
pvc = s.get("kubernetesStatus", {}).get("pvcName") or name
|
||||
if s.get("robustness") != "healthy":
|
||||
problems.append(f"{pvc} is {s.get('robustness')}")
|
||||
last = newest.get(name)
|
||||
if not last:
|
||||
problems.append(f"{pvc} has no backup")
|
||||
elif (NOW - parse(last)).total_seconds() > max_hours * 3600:
|
||||
problems.append(f"{pvc} last backed up {age(last)} ago")
|
||||
return "; ".join(problems) or None
|
||||
|
||||
|
||||
def vault():
|
||||
pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver")
|
||||
if not pods:
|
||||
return "no Vault pods"
|
||||
sealed = [p["metadata"]["name"] for p in pods
|
||||
if p["metadata"].get("labels", {}).get("vault-sealed") != "false"]
|
||||
return f"sealed: {', '.join(sealed)}" if sealed else None
|
||||
|
||||
|
||||
def argocd():
|
||||
bad = []
|
||||
for a in items("/apis/argoproj.io/v1alpha1/namespaces/argocd/applications"):
|
||||
s = a.get("status", {})
|
||||
sync = s.get("sync", {}).get("status")
|
||||
health = s.get("health", {}).get("status")
|
||||
if sync != "Synced" or health != "Healthy":
|
||||
bad.append(f"{a['metadata']['name']} {sync}/{health}")
|
||||
return "; ".join(bad) or None
|
||||
|
||||
|
||||
def postgres():
|
||||
bad = [f"{c['metadata']['namespace']}/{c['metadata']['name']}: {c.get('status', {}).get('phase')}"
|
||||
for c in items("/apis/postgresql.cnpg.io/v1/clusters")
|
||||
if c.get("status", {}).get("phase") != "Cluster in healthy state"]
|
||||
return "; ".join(bad) or None
|
||||
|
||||
|
||||
def push(key, error):
|
||||
query = {"success": "false" if error else "true"}
|
||||
if error:
|
||||
query["error"] = error[:500]
|
||||
url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}"
|
||||
req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
r.read()
|
||||
|
||||
|
||||
checks = {
|
||||
"cluster_velero-daily": lambda: velero("daily-backup", 26),
|
||||
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
|
||||
"cluster_longhorn": lambda: longhorn(26),
|
||||
"cluster_vault-unsealed": vault,
|
||||
"cluster_argocd-apps": argocd,
|
||||
"cluster_postgres": postgres,
|
||||
}
|
||||
failed_pushes = 0
|
||||
for key, check in checks.items():
|
||||
try:
|
||||
error = check()
|
||||
except Exception as e: # report a broken check as a failure
|
||||
error = f"check failed: {e}"
|
||||
print(f"{key}: {error or 'ok'}", flush=True)
|
||||
try:
|
||||
push(key, error)
|
||||
except Exception as e:
|
||||
print(f"{key}: push to Gatus failed: {e}", flush=True)
|
||||
failed_pushes += 1
|
||||
sys.exit(1 if failed_pushes else 0)
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: backup-check
|
||||
spec:
|
||||
schedule: "17 * * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 300
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup-check
|
||||
spec:
|
||||
serviceAccountName: backup-check
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: check
|
||||
image: python:3.15.1-alpine3.24
|
||||
command: ["python", "/script/check.py"]
|
||||
env:
|
||||
- name: PYTHONDONTWRITEBYTECODE
|
||||
value: "1"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: backup-check-token
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /script
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: backup-check
|
||||
+46
-1
@@ -123,7 +123,7 @@ endpoints:
|
||||
conditions: *http-ok
|
||||
- name: foto.scoutingmbg.nl
|
||||
group: Scouting MBG
|
||||
url: https://foto.scoutingmbg.nl/_app
|
||||
url: https://foto.scoutingmbg.nl/api/server/ping
|
||||
interval: 60s
|
||||
conditions: *http-ok
|
||||
|
||||
@@ -136,3 +136,48 @@ endpoints:
|
||||
- "[STATUS] == 200"
|
||||
- "[BODY].status == UP"
|
||||
alerts: *ntfy
|
||||
|
||||
# Pushed every hour by the backup-check CronJob (backup-check.yaml), which
|
||||
# looks at what can't be seen from outside. s2 alerts on these itself: s1
|
||||
# has no view into the cluster.
|
||||
external-endpoints:
|
||||
- name: velero-daily
|
||||
group: cluster
|
||||
token: ${BACKUP_CHECK_TOKEN}
|
||||
heartbeat:
|
||||
interval: 3h
|
||||
alerts: &cluster-alert
|
||||
- type: ntfy
|
||||
failure-threshold: 2
|
||||
success-threshold: 1
|
||||
send-on-resolved: true
|
||||
- name: velero-weekly
|
||||
group: cluster
|
||||
token: ${BACKUP_CHECK_TOKEN}
|
||||
heartbeat:
|
||||
interval: 3h
|
||||
alerts: *cluster-alert
|
||||
- name: longhorn
|
||||
group: cluster
|
||||
token: ${BACKUP_CHECK_TOKEN}
|
||||
heartbeat:
|
||||
interval: 3h
|
||||
alerts: *cluster-alert
|
||||
- name: vault-unsealed
|
||||
group: cluster
|
||||
token: ${BACKUP_CHECK_TOKEN}
|
||||
heartbeat:
|
||||
interval: 3h
|
||||
alerts: *cluster-alert
|
||||
- name: argocd-apps
|
||||
group: cluster
|
||||
token: ${BACKUP_CHECK_TOKEN}
|
||||
heartbeat:
|
||||
interval: 3h
|
||||
alerts: *cluster-alert
|
||||
- name: postgres
|
||||
group: cluster
|
||||
token: ${BACKUP_CHECK_TOKEN}
|
||||
heartbeat:
|
||||
interval: 3h
|
||||
alerts: *cluster-alert
|
||||
@@ -25,6 +25,9 @@ spec:
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: gatus-ntfy
|
||||
# Token for the external endpoints, see backup-check.yaml
|
||||
- secretRef:
|
||||
name: backup-check-token
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
|
||||
@@ -10,6 +10,8 @@ resources:
|
||||
- gatus-service.yaml
|
||||
- certificate.yaml
|
||||
- httproute.yaml
|
||||
- networkpolicy.yaml
|
||||
- backup-check.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: gatus-config
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# Gatus faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces.
|
||||
# Its checks go to public addresses; alerts go to ntfy in-cluster.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8080
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: ntfy
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: ntfy
|
||||
ports:
|
||||
- port: 80
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# backup-check.yaml reads backups, volumes and Vault's pods from the API
|
||||
# server (the node, port 16443).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: backup-check-to-api
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: backup-check
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -20,7 +20,7 @@ spec:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: gitea
|
||||
image: gitea/gitea:1.27.3
|
||||
image: gitea/gitea:28.2.0
|
||||
env:
|
||||
- name: USER_UID
|
||||
value: "1000"
|
||||
@@ -50,6 +50,27 @@ spec:
|
||||
- containerPort: 3000
|
||||
name: http
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 336Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
# /api/healthz checks the database and the cache.
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/healthz
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/healthz
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# Gitea faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces.
|
||||
# Mirrors and webhooks go out over the internet.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 3000
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
+45
-22
@@ -15,8 +15,10 @@ spec:
|
||||
project: default
|
||||
sources:
|
||||
- chart: immich
|
||||
repoURL: https://immich-app.github.io/immich-charts
|
||||
targetRevision: 0.10.3
|
||||
# No scheme, so Argo CD pulls the chart from the OCI registry. Charts
|
||||
# after 0.12.0 are only published there.
|
||||
repoURL: ghcr.io/immich-app/immich-charts
|
||||
targetRevision: 0.13.4
|
||||
helm:
|
||||
values: |
|
||||
controllers:
|
||||
@@ -24,7 +26,7 @@ spec:
|
||||
containers:
|
||||
main:
|
||||
image:
|
||||
tag: v2.7.5
|
||||
tag: v3.3.5
|
||||
env:
|
||||
REDIS_HOSTNAME: '{{ printf "%s-valkey" .Release.Name }}'
|
||||
IMMICH_MACHINE_LEARNING_URL: '{{ printf "http://%s-machine-learning:3003" .Release.Name }}'
|
||||
@@ -48,16 +50,10 @@ spec:
|
||||
# Automatically creating the library volume is not supported by this chart
|
||||
# You have to specify an existing PVC to use
|
||||
existingClaim: immich-claim
|
||||
# configuration is immich-config.json converted to yaml
|
||||
# ref: https://immich.app/docs/install/config-file/
|
||||
#
|
||||
configuration: {}
|
||||
# trash:
|
||||
# enabled: false
|
||||
# days: 30
|
||||
# storageTemplate:
|
||||
# enabled: true
|
||||
# template: "{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}"
|
||||
# The config file is a Secret rendered by ESO, since it holds the
|
||||
# OAuth client secret (immich/immich-config.yaml).
|
||||
configurationKind: Secret
|
||||
existingConfiguration: immich-config
|
||||
|
||||
# Dependencies
|
||||
|
||||
@@ -67,18 +63,22 @@ spec:
|
||||
main:
|
||||
containers:
|
||||
main:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
image:
|
||||
repository: docker.io/valkey/valkey
|
||||
tag: 9.0-alpine@sha256:b4ee67d73e00393e712accc72cfd7003b87d0fcd63f0eba798b23251bfc9c394
|
||||
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
||||
pullPolicy: IfNotPresent
|
||||
persistence:
|
||||
data:
|
||||
enabled: true
|
||||
size: 1Gi
|
||||
# Optional: Set this to persistentVolumeClaim to keep job queues persistent
|
||||
# Optional: to keep job queues persistent, use a
|
||||
# persistentVolumeClaim with size, accessMode and storageClass
|
||||
type: emptyDir
|
||||
accessMode: ReadWriteOnce
|
||||
storageClass: longhorn
|
||||
|
||||
# Immich components
|
||||
|
||||
@@ -91,6 +91,12 @@ spec:
|
||||
image:
|
||||
repository: ghcr.io/immich-app/immich-server
|
||||
pullPolicy: IfNotPresent
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 1280Mi
|
||||
limits:
|
||||
memory: 3Gi
|
||||
# Node trusts its built-in roots, which still include the
|
||||
# ones above the compromised eMudhra intermediates. Use
|
||||
# patched-ca-bundle through OpenSSL until Node ships a fix.
|
||||
@@ -115,10 +121,21 @@ spec:
|
||||
image:
|
||||
repository: ghcr.io/immich-app/immich-machine-learning
|
||||
pullPolicy: IfNotPresent
|
||||
# Idle at 240Mi; 884Mi with CLIP, face recognition and
|
||||
# OCR loaded (2026-10-28, a test image through each).
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
env:
|
||||
TRANSFORMERS_CACHE: /cache
|
||||
HF_XET_CACHE: /cache/huggingface-xet
|
||||
MPLCONFIGDIR: /cache/matplotlib-config
|
||||
# The optimized models of v3.3.0: faster, less memory,
|
||||
# and the same results, so no jobs to run again.
|
||||
MACHINE_LEARNING_MODEL_REVISION: v2
|
||||
# Trust store without the compromised eMudhra
|
||||
# intermediates; requests would otherwise use certifi.
|
||||
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
||||
@@ -131,13 +148,19 @@ spec:
|
||||
globalMounts:
|
||||
- path: /etc/ssl/certs
|
||||
readOnly: true
|
||||
# The models and their one-off preparation, kept across restarts.
|
||||
# Downloaded again if lost, so no backups: the no-backup group
|
||||
# keeps it out of Longhorn's default one (backup-daily), and
|
||||
# Velero skips that label too (velero/skip-emptydir-policy.yaml).
|
||||
cache:
|
||||
enabled: true
|
||||
type: persistentVolumeClaim
|
||||
storageClass: longhorn
|
||||
accessMode: ReadWriteOnce
|
||||
size: 10Gi
|
||||
# Optional: Set this to persistentVolumeClaim to avoid downloading the ML models every start.
|
||||
type: emptyDir
|
||||
accessMode: ReadWriteMany
|
||||
# storageClass: longhorn
|
||||
labels:
|
||||
recurring-job.longhorn.io/source: enabled
|
||||
recurring-job-group.longhorn.io/no-backup: enabled
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: httpd-conf
|
||||
data:
|
||||
httpd.conf: |
|
||||
ServerRoot "/usr/local/apache2"
|
||||
Listen 8002
|
||||
LoadModule mpm_event_module modules/mod_mpm_event.so
|
||||
LoadModule authn_core_module modules/mod_authn_core.so
|
||||
LoadModule authz_core_module modules/mod_authz_core.so
|
||||
LoadModule xml2enc_module modules/mod_xml2enc.so
|
||||
LoadModule proxy_html_module modules/mod_proxy_html.so
|
||||
LoadModule log_config_module modules/mod_log_config.so
|
||||
LoadModule headers_module modules/mod_headers.so
|
||||
LoadModule proxy_module modules/mod_proxy.so
|
||||
LoadModule proxy_http_module modules/mod_proxy_http.so
|
||||
LoadModule http2_module modules/mod_http2.so
|
||||
LoadModule proxy_http2_module modules/mod_proxy_http2.so
|
||||
LoadModule unixd_module modules/mod_unixd.so
|
||||
LoadModule authopenid_module /usr/lib/apache2/modules/mod_auth_openid.so
|
||||
LoadModule dir_module modules/mod_dir.so
|
||||
LoadModule alias_module modules/mod_alias.so
|
||||
LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so
|
||||
LoadModule rewrite_module modules/mod_rewrite.so
|
||||
|
||||
MaxConnectionsPerChild 30
|
||||
|
||||
<IfModule unixd_module>
|
||||
User www-data
|
||||
Group www-data
|
||||
</IfModule>
|
||||
|
||||
ServerAdmin webmaster@scoutingmbg.nl
|
||||
ServerName foto.scoutingmbg.nl
|
||||
|
||||
<Directory />
|
||||
AllowOverride none
|
||||
Require all denied
|
||||
</Directory>
|
||||
|
||||
DocumentRoot "/usr/local/apache2/htdocs"
|
||||
|
||||
<Directory "/usr/local/apache2/htdocs">
|
||||
Options Indexes FollowSymLinks
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
<Location />
|
||||
AuthType OpenID
|
||||
require valid-user
|
||||
|
||||
AuthOpenIDDBLocation /tmp/mod_auth_openid.db
|
||||
AuthOpenIDTrusted ^https://login.scouting.nl/.*$ ^https://login.scouting.nl/user/$
|
||||
AuthOpenIDUseCookie On
|
||||
AuthOpenIDCookieName open_id_session_id
|
||||
AuthOpenIDCookieLifespan 31536000
|
||||
AuthOpenIDServerName https://foto.scoutingmbg.nl
|
||||
AuthOpenIDSingleIdP https://login.scouting.nl/user/
|
||||
AuthOpenIDSecureCookie On
|
||||
</Location>
|
||||
|
||||
<Location "/_app">
|
||||
AuthType None
|
||||
Require all granted
|
||||
</Location>
|
||||
|
||||
AllowEncodedSlashes NoDecode
|
||||
ProxyPreserveHost Off
|
||||
|
||||
RewriteEngine on
|
||||
RewriteRule ^/?photos/?$ /sharing [R=301,L]
|
||||
|
||||
ProxyPass / http://helm-immich-server:2283/
|
||||
ProxyPassReverse / http://helm-immich-server:2283/
|
||||
RewriteEngine on
|
||||
RewriteCond %{HTTP:Upgrade} websocket [NC]
|
||||
RewriteCond %{HTTP:Connection} upgrade [NC]
|
||||
RewriteRule ^/?(.*) "ws://helm-immich-server:2283/$1" [P,L]
|
||||
|
||||
|
||||
# must be bigger than pingInterval (25s by default) + pingTimeout (20s by default)
|
||||
ProxyTimeout 60
|
||||
ProxyRequests off
|
||||
|
||||
Header always set Access-Control-Allow-Origin "*"
|
||||
Header always set Access-Control-Max-Age "1000"
|
||||
Header always set Access-Control-Allow-Headers "x-requested-with, Content-Type, origin, authorization, accept, client-security-token"
|
||||
|
||||
<IfModule dir_module>
|
||||
DirectoryIndex index.html
|
||||
</IfModule>
|
||||
|
||||
<Files ".ht*">
|
||||
Require all denied
|
||||
</Files>
|
||||
|
||||
ErrorLog /proc/self/fd/2
|
||||
LogLevel warn
|
||||
|
||||
<IfModule log_config_module>
|
||||
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
|
||||
LogFormat "%h %l %u %t \"%r\" %>s %b" common
|
||||
<IfModule logio_module>
|
||||
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
|
||||
</IfModule>
|
||||
CustomLog /proc/self/fd/1 common
|
||||
</IfModule>
|
||||
|
||||
<IfModule alias_module>
|
||||
ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/"
|
||||
</IfModule>
|
||||
<Directory "/usr/local/apache2/cgi-bin">
|
||||
AllowOverride None
|
||||
Options None
|
||||
Require all granted
|
||||
</Directory>
|
||||
<IfModule headers_module>
|
||||
RequestHeader unset Proxy early
|
||||
</IfModule>
|
||||
<IfModule mime_module>
|
||||
TypesConfig conf/mime.types
|
||||
AddType application/x-compress .Z
|
||||
AddType application/x-gzip .gz .tgz
|
||||
</IfModule>
|
||||
<IfModule proxy_html_module>
|
||||
Include conf/extra/proxy-html.conf
|
||||
</IfModule>
|
||||
<IfModule ssl_module>
|
||||
SSLRandomSeed startup builtin
|
||||
SSLRandomSeed connect builtin
|
||||
</IfModule>
|
||||
@@ -9,10 +9,12 @@ spec:
|
||||
sectionName: https-foto
|
||||
hostnames:
|
||||
- foto.scoutingmbg.nl
|
||||
# Straight to immich, which logs people in with Scouts Online through
|
||||
# authentik (docs/immich-scouts-online.md).
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: sol-auth-svc
|
||||
port: 8002
|
||||
- name: helm-immich-server
|
||||
port: 2283
|
||||
# No limit on the whole request, as with nginx. Envoy's default of 15s
|
||||
# cuts off uploads and long-lived connections.
|
||||
timeouts:
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# immich's settings, rendered with the OAuth client secret from Vault. With
|
||||
# a config file the settings are read-only in the admin UI. `{{`...`}}` keeps
|
||||
# immich's own template syntax away from ESO's.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: immich-config
|
||||
namespace: immich
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: immich-config
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
engineVersion: v2
|
||||
data:
|
||||
immich-config.yaml: |
|
||||
server:
|
||||
externalDomain: https://foto.scoutingmbg.nl
|
||||
storageTemplate:
|
||||
enabled: true
|
||||
template: "{{`{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}`}}"
|
||||
# authentik passes on the Scouts Online login and sets the
|
||||
# immich_role and immich_quota claims (authentik/blueprint-scouting.yaml).
|
||||
oauth:
|
||||
enabled: true
|
||||
issuerUrl: https://auth.scoutingmbg.nl/application/o/immich/
|
||||
clientId: immich
|
||||
clientSecret: "{{ .client_secret }}"
|
||||
scope: openid email profile immich
|
||||
buttonText: Inloggen met Scouts Online
|
||||
autoRegister: true
|
||||
# /auth/login?autoLaunch=0 still shows the login page
|
||||
autoLaunch: true
|
||||
# Admins log in through authentik too. To get back in without it,
|
||||
# enable this again (immich-admin can't change a config file).
|
||||
passwordLogin:
|
||||
enabled: false
|
||||
data:
|
||||
- secretKey: client_secret
|
||||
remoteRef:
|
||||
key: kv/immich-oidc
|
||||
property: client_secret
|
||||
@@ -0,0 +1,135 @@
|
||||
# immich faces the internet (every member of the group logs in, and shared
|
||||
# links are public), so its pods only reach each other, DNS and the
|
||||
# internet: not the LAN (NAS, router), the Kubernetes API or other
|
||||
# namespaces. The OIDC login with authentik goes out through its public
|
||||
# address. Ingress comes from Envoy, the CNPG operator and the node itself
|
||||
# (kubelet probes; 10.1.7.64 is the node's Calico VXLAN address,
|
||||
# projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
# The internet: authentik's public address, the ML models from Hugging
|
||||
# Face.
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: server-from-envoy
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: helm-immich
|
||||
app.kubernetes.io/name: server
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 2283
|
||||
---
|
||||
# The CNPG operator checks each instance on its status port and connects to
|
||||
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
||||
# through the API server (the node, port 16443). The barman cloud sidecar
|
||||
# sends the WAL and the base backups to Garage on the NAS
|
||||
# (postgresql-backup.yaml).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: postgres-cnpg
|
||||
spec:
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: cnpg.io/cluster
|
||||
operator: Exists
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: cnpg-system
|
||||
ports:
|
||||
- port: 8000
|
||||
- port: 5432
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.99/32
|
||||
ports:
|
||||
- port: 3900
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -0,0 +1,99 @@
|
||||
# Backups of immich's database (albums, faces, the photos' metadata; the
|
||||
# photos themselves are files on immich-claim) by CNPG itself, through the
|
||||
# barman cloud plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base
|
||||
# backup every night, and every WAL segment as soon as it is full or 5
|
||||
# minutes old (archive_timeout). Both go to the bucket cnpg-immich in Garage
|
||||
# on the NAS (nas/garage), so the database can be restored to any moment of
|
||||
# the last 30 days. Velero copies the data directory while PostgreSQL runs,
|
||||
# which is not a consistent copy. Only the PostgreSQL 17 cluster; the 14
|
||||
# one is about to go.
|
||||
# The key is Garage's cnpg-immich, in Vault as kv/cnpg-immich-s3; it can
|
||||
# only reach this bucket.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: immich-postgres-17-s3
|
||||
namespace: immich
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: immich-postgres-17-s3
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
engineVersion: v2
|
||||
data:
|
||||
ACCESS_KEY_ID: "{{ .access_key_id }}"
|
||||
ACCESS_SECRET_KEY: "{{ .secret_access_key }}"
|
||||
# Garage rejects requests signed for another region
|
||||
REGION: garage
|
||||
data:
|
||||
- secretKey: access_key_id
|
||||
remoteRef:
|
||||
key: kv/cnpg-immich-s3
|
||||
property: access_key_id
|
||||
- secretKey: secret_access_key
|
||||
remoteRef:
|
||||
key: kv/cnpg-immich-s3
|
||||
property: secret_access_key
|
||||
---
|
||||
apiVersion: barmancloud.cnpg.io/v1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: immich-postgres-17
|
||||
namespace: immich
|
||||
annotations:
|
||||
# The CRD comes with the plugin's own app
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
retentionPolicy: 30d
|
||||
configuration:
|
||||
destinationPath: s3://cnpg-immich/
|
||||
endpointURL: http://192.168.1.99:3900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: immich-postgres-17-s3
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: immich-postgres-17-s3
|
||||
key: ACCESS_SECRET_KEY
|
||||
region:
|
||||
name: immich-postgres-17-s3
|
||||
key: REGION
|
||||
wal:
|
||||
compression: gzip
|
||||
data:
|
||||
compression: gzip
|
||||
instanceSidecarConfiguration:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
env:
|
||||
# Newer boto3 adds checksums that S3-compatible stores can reject;
|
||||
# the plugin's docs suggest sending them only when required.
|
||||
- name: AWS_REQUEST_CHECKSUM_CALCULATION
|
||||
value: when_required
|
||||
- name: AWS_RESPONSE_CHECKSUM_VALIDATION
|
||||
value: when_required
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: ScheduledBackup
|
||||
metadata:
|
||||
name: immich-postgres-17-daily
|
||||
namespace: immich
|
||||
spec:
|
||||
cluster:
|
||||
name: immich-postgres-17
|
||||
# 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right
|
||||
# away, so WAL archiving has a base backup to start from.
|
||||
schedule: "0 0 3 * * *"
|
||||
immediate: true
|
||||
backupOwnerReference: self
|
||||
method: plugin
|
||||
pluginConfiguration:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
@@ -9,11 +9,24 @@ spec:
|
||||
# CNPG image with VectorChord and pgvector, which immich 2.x and 3.x use
|
||||
# for its vector indexes (pgvecto.rs is gone in immich 3.0).
|
||||
imageName: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 320Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
|
||||
postgresql:
|
||||
shared_preload_libraries:
|
||||
- "vchord.so"
|
||||
|
||||
# WAL archiving and base backups to Garage, see postgresql-backup.yaml
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: immich-postgres-17
|
||||
|
||||
# Filled once from the PostgreSQL 14 cluster (immich-postgres) after its
|
||||
# embedding columns were turned into real[] and pgvecto.rs was dropped,
|
||||
# see docs/upgrade-k8s-1.33.md.
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
# Shares the Scouting MBG photos with every immich user. immich has no user
|
||||
# groups (on its roadmap as "Soon"), so this does it per user:
|
||||
# - albums that uploaders (kaderleden) create get the group account as
|
||||
# editor, unless their name starts with "Privé" or "Prive";
|
||||
# - every album the group account owns or edits gets every user, as editor
|
||||
# when they may upload (quota other than 0, which authentik sets from
|
||||
# Scouts Online) and as viewer otherwise;
|
||||
# - the group account's library is shared as a partner and shown in
|
||||
# everyone's timeline, so search covers it (immich only searches your own
|
||||
# and your timeline partners' photos);
|
||||
# - the invitation notifications for albums older than a week are hidden,
|
||||
# or a new member would get one per album;
|
||||
# - sessions of OAuth users older than SESSION_MAX_AGE_DAYS are ended, so
|
||||
# someone who left the group loses access at the next login at Scouts
|
||||
# Online.
|
||||
# Albums and partners go through the API. Joining other people's albums
|
||||
# (the API only lets an album's owner or editors share it), the timeline
|
||||
# flag (which only the receiving user can set), the notifications and the
|
||||
# sessions go straight to the database.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: share-sync
|
||||
namespace: immich
|
||||
data:
|
||||
sync.py: |
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import urllib.request
|
||||
|
||||
API = os.environ["IMMICH_URL"].rstrip("/") + "/api"
|
||||
KEY = os.environ["IMMICH_API_KEY"]
|
||||
GROUP_EMAIL = os.environ["GROUP_EMAIL"]
|
||||
SKIP_EMAILS = set(os.environ.get("SKIP_EMAILS", "").split())
|
||||
SESSION_MAX_AGE_DAYS = int(os.environ.get("SESSION_MAX_AGE_DAYS", "30"))
|
||||
DRY_RUN = os.environ.get("DRY_RUN") == "1"
|
||||
|
||||
|
||||
def api(method, path, body=None):
|
||||
req = urllib.request.Request(
|
||||
API + path,
|
||||
method=method,
|
||||
data=None if body is None else json.dumps(body).encode(),
|
||||
headers={"x-api-key": KEY, "Accept": "application/json", "Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data) if data else None
|
||||
|
||||
|
||||
def sql(query, **params):
|
||||
"""Runs query through psql; :'name' in it is params[name], quoted."""
|
||||
args = ["psql", "-X", "-q", "-At", "-F", "\t", "-v", "ON_ERROR_STOP=1"]
|
||||
for name, value in params.items():
|
||||
args += ["-v", f"{name}={value}"]
|
||||
out = subprocess.run(args, input=query, capture_output=True, text=True, check=True).stdout
|
||||
return [line.split("\t") for line in out.splitlines() if line]
|
||||
|
||||
|
||||
def change(what, call):
|
||||
print(("would " if DRY_RUN else "") + what, flush=True)
|
||||
if not DRY_RUN:
|
||||
call()
|
||||
|
||||
|
||||
users = sql('SELECT id, email, name, coalesce("quotaSizeInBytes"::text, \'\') FROM "user" WHERE "deletedAt" IS NULL')
|
||||
group = next(u for u in users if u[1] == GROUP_EMAIL)
|
||||
group_id, group_name = group[0], group[2]
|
||||
members = {u[0]: u[1] for u in users if u[0] != group_id and u[1] not in SKIP_EMAILS}
|
||||
# A quota of 0 bytes is how immich (and authentik's immich_quota claim)
|
||||
# says: no uploads.
|
||||
wanted = {u[0]: "viewer" if u[3] == "0" else "editor" for u in users if u[0] in members}
|
||||
|
||||
# Albums of uploaders that the group account isn't in yet
|
||||
UNJOINED = """
|
||||
FROM album_user au
|
||||
JOIN "user" u ON u.id = au."userId"
|
||||
JOIN album a ON a.id = au."albumId"
|
||||
WHERE au.role = 'owner' AND u.id != :'group'
|
||||
AND u."deletedAt" IS NULL AND a."deletedAt" IS NULL
|
||||
AND (u."quotaSizeInBytes" IS NULL OR u."quotaSizeInBytes" > 0)
|
||||
AND a."albumName" !~* '^\\s*priv[eé]'
|
||||
AND NOT EXISTS (SELECT 1 FROM album_user g WHERE g."albumId" = a.id AND g."userId" = :'group')
|
||||
"""
|
||||
if DRY_RUN:
|
||||
for (name,) in sql("SELECT a.\"albumName\"" + UNJOINED, group=group_id):
|
||||
print(f"would join {name}", flush=True)
|
||||
else:
|
||||
for (name,) in sql(
|
||||
f"""
|
||||
WITH joined AS (
|
||||
INSERT INTO album_user ("albumId", "userId", role)
|
||||
SELECT a.id, :'group', 'editor' {UNJOINED}
|
||||
RETURNING "albumId"
|
||||
)
|
||||
SELECT a."albumName" FROM joined JOIN album a ON a.id = joined."albumId"
|
||||
""",
|
||||
group=group_id,
|
||||
):
|
||||
print(f"join {name}", flush=True)
|
||||
|
||||
for album in api("GET", "/albums"):
|
||||
roles = {au["user"]["id"]: au["role"] for au in album["albumUsers"]}
|
||||
if roles.get(group_id) not in ("owner", "editor"):
|
||||
continue
|
||||
add = [{"userId": uid, "role": role} for uid, role in wanted.items() if uid not in roles]
|
||||
if add:
|
||||
change(
|
||||
f"add {len(add)} users to {album['albumName']}",
|
||||
lambda: api("PUT", f"/albums/{album['id']}/users", {"albumUsers": add}),
|
||||
)
|
||||
for uid, role in wanted.items():
|
||||
if roles.get(uid) in ("viewer", "editor") and roles[uid] != role:
|
||||
change(
|
||||
f"make {members[uid]} {role} of {album['albumName']}",
|
||||
lambda: api("PUT", f"/albums/{album['id']}/user/{uid}", {"role": role}),
|
||||
)
|
||||
|
||||
partners = {p["id"] for p in api("GET", "/partners?direction=shared-by")}
|
||||
for uid, email in members.items():
|
||||
if uid not in partners:
|
||||
change(f"share the library with {email}", lambda: api("POST", "/partners", {"sharedWithId": uid}))
|
||||
|
||||
if DRY_RUN:
|
||||
raise SystemExit(0)
|
||||
|
||||
sql(
|
||||
'UPDATE partner SET "inTimeline" = true WHERE "sharedById" = :\'group\' AND NOT "inTimeline"',
|
||||
group=group_id,
|
||||
)
|
||||
sql(
|
||||
"""
|
||||
UPDATE notification SET "deletedAt" = now()
|
||||
WHERE type = 'AlbumInvite' AND "deletedAt" IS NULL AND "readAt" IS NULL
|
||||
AND description LIKE :'sender' || ' shared an album%'
|
||||
-- immich stores data as a JSON string inside the jsonb
|
||||
AND ((data #>> '{}')::jsonb ->> 'albumId')::uuid IN (SELECT id FROM album WHERE "createdAt" < now() - interval '7 days')
|
||||
""",
|
||||
sender=group_name,
|
||||
)
|
||||
sql(
|
||||
"""
|
||||
DELETE FROM session
|
||||
WHERE "createdAt" < now() - make_interval(days => :days)
|
||||
AND "userId" IN (SELECT id FROM "user" WHERE "oauthId" IS NOT NULL)
|
||||
""",
|
||||
days=SESSION_MAX_AGE_DAYS,
|
||||
)
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: immich-share
|
||||
namespace: immich
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: immich-share
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
# API key of the group account, with album.read, albumUser.create,
|
||||
# albumUser.update, partner.read and partner.create
|
||||
- secretKey: api_key
|
||||
remoteRef:
|
||||
key: kv/immich-share
|
||||
property: api_key
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: share-sync
|
||||
namespace: immich
|
||||
spec:
|
||||
schedule: "*/10 * * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 1800
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 26
|
||||
containers:
|
||||
- name: sync
|
||||
# The database's own image: it has python3 and psql.
|
||||
image: ghcr.io/tensorchord/cloudnative-vectorchord:17.11-1.1.1
|
||||
command: ["python3", "/script/sync.py"]
|
||||
env:
|
||||
- name: IMMICH_URL
|
||||
value: http://helm-immich-server:2283
|
||||
- name: IMMICH_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: immich-share
|
||||
key: api_key
|
||||
- name: GROUP_EMAIL
|
||||
value: fotoupload@scoutingmbg.nl
|
||||
# The shared viewing account from before Scouts Online
|
||||
- name: SKIP_EMAILS
|
||||
value: foto@scoutingmbg.nl
|
||||
- name: PGHOST
|
||||
value: immich-postgres-17-rw
|
||||
- name: PGSSLMODE
|
||||
value: require
|
||||
# album names are UTF-8 (and so is the Privé check)
|
||||
- name: PGCLIENTENCODING
|
||||
value: UTF8
|
||||
- name: PGDATABASE
|
||||
value: immich
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: immich-postgres-17-app
|
||||
key: username
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: immich-postgres-17-app
|
||||
key: password
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /script
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: share-sync
|
||||
@@ -1,40 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: sol-auth
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: sol-auth
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: sol-auth
|
||||
spec:
|
||||
# securityContext:
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 1000
|
||||
# fsGroup: 1000
|
||||
containers:
|
||||
- image: hebury/sol-auth:2.3
|
||||
name: sol-auth
|
||||
resources:
|
||||
requests:
|
||||
memory: "50Mi"
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8002
|
||||
hostPort: 8002
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
mountPath: /config/
|
||||
volumes:
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: httpd-conf
|
||||
restartPolicy: Always
|
||||
terminationGracePeriodSeconds: 5
|
||||
@@ -1,13 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: sol-auth-svc
|
||||
spec:
|
||||
ports:
|
||||
- name: http-svc
|
||||
port: 8002
|
||||
targetPort: 8002
|
||||
selector:
|
||||
app: sol-auth
|
||||
status:
|
||||
loadBalancer: {}
|
||||
@@ -1,13 +0,0 @@
|
||||
FROM httpd:2.4-bullseye
|
||||
RUN apt-get update && \
|
||||
apt-get install --no-install-recommends -y \
|
||||
ca-certificates libapache2-mod-auth-openid
|
||||
RUN touch /tmp/mod_auth_openid.db && chown www-data /tmp/mod_auth_openid.db
|
||||
RUN mkdir /config
|
||||
RUN touch /config/httpd.conf
|
||||
RUN rm /usr/local/apache2/conf/httpd.conf
|
||||
RUN ln -s /config/httpd.conf /usr/local/apache2/conf/httpd.conf
|
||||
|
||||
# how to build: docker build . -t hebury/sol-auth:2.3
|
||||
# how to push: docker push hebury/sol-auth:2.3
|
||||
# do not remember how to build for arm atm.
|
||||
@@ -3,3 +3,4 @@ kind: Kustomization
|
||||
resources:
|
||||
- operator-crds-helm.yaml
|
||||
- operator-helm.yaml
|
||||
- networkpolicy.yaml
|
||||
@@ -0,0 +1,54 @@
|
||||
# The mariadb-operator's three pods reach each other, DNS and the API
|
||||
# server (the node, port 16443); the operator also connects to MariaDB on
|
||||
# 3306 for the databases, users and grants it manages (mbgwp's
|
||||
# mariadb-from-operator lets it in). Besides their own namespace they take
|
||||
# connections only from the node: the API server calls the webhook from
|
||||
# there and the kubelet probes come from there too (10.1.7.64 is the node's
|
||||
# Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr). This
|
||||
# folder's Application deploys into argocd (the operator's own
|
||||
# Applications), hence the namespace here.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
namespace: mariadb-system
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- namespaceSelector: {}
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: mariadb
|
||||
ports:
|
||||
- port: 3306
|
||||
@@ -13,7 +13,7 @@ spec:
|
||||
source:
|
||||
chart: mariadb-operator-crds
|
||||
repoURL: https://mariadb-operator.github.io/mariadb-operator
|
||||
targetRevision: 25.10.4
|
||||
targetRevision: 26.10.2
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
|
||||
@@ -13,17 +13,35 @@ spec:
|
||||
source:
|
||||
chart: mariadb-operator
|
||||
repoURL: https://mariadb-operator.github.io/mariadb-operator
|
||||
targetRevision: 25.10.4
|
||||
targetRevision: 26.10.2
|
||||
helm:
|
||||
values: |
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
metrics:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
webhook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
certController:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
syncPolicy:
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# WordPress faces the internet, so if it is ever taken over it should only
|
||||
# reach its own database, DNS and the internet: not the LAN (NAS, router),
|
||||
# the Kubernetes API or other namespaces. Ingress comes from Envoy, the
|
||||
# mariadb-operator and the node itself (kubelet probes; 10.1.7.64 is the
|
||||
# node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
# The internet, which includes this site's own public address
|
||||
# (wp-cron's loopback requests go out and come back through Envoy).
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: wordpress-from-envoy
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wordpress
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8000
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: mariadb-from-operator
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: mariadb
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: mariadb-system
|
||||
ports:
|
||||
- port: 3306
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -0,0 +1,42 @@
|
||||
# Garage on the NAS
|
||||
|
||||
S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
|
||||
the cluster. Replaced MinIO, which is no longer maintained. One node, data
|
||||
in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
|
||||
3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
|
||||
|
||||
The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
|
||||
in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
|
||||
its PATH.
|
||||
|
||||
## Setup
|
||||
|
||||
`garage.toml` here has placeholders for the two secrets; on the NAS they
|
||||
were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
|
||||
`openssl rand -base64 32` (`admin_token`), and only exist there.
|
||||
|
||||
```
|
||||
cd /volume1/docker/garage
|
||||
docker compose up -d
|
||||
g="docker exec garage /garage"
|
||||
$g layout assign -z nas -c 800G <node id from $g status>
|
||||
$g layout apply --version 1
|
||||
$g bucket create velero-backups
|
||||
$g key create velero
|
||||
$g bucket allow --read --write --key velero velero-backups
|
||||
```
|
||||
|
||||
Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
|
||||
`secret_access_key`) and reaches Velero through
|
||||
`velero/external-secret.yaml`. Without the cluster, `$g key info velero
|
||||
--show-secret` on the NAS shows it again.
|
||||
|
||||
CNPG's backups (`authentik/postgresql-backup.yaml`,
|
||||
`immich/postgresql-backup.yaml`) have a bucket and a key per cluster, so
|
||||
neither database can touch the other's backups. Made on 2026-10-26 the
|
||||
same way as Velero's, with `cnpg-authentik` and `cnpg-immich` as both
|
||||
bucket and key name; the keys are in Vault as `kv/cnpg-authentik-s3` and
|
||||
`kv/cnpg-immich-s3`.
|
||||
|
||||
Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
|
||||
after merging, `docker compose pull && docker compose up -d` on the NAS.
|
||||
@@ -0,0 +1,12 @@
|
||||
services:
|
||||
garage:
|
||||
image: dxflrs/garage:v2.5.0
|
||||
container_name: garage
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3900:3900" # S3 API
|
||||
- "3903:3903" # admin API
|
||||
volumes:
|
||||
- ./garage.toml:/etc/garage.toml:ro
|
||||
- ./meta:/var/lib/garage/meta
|
||||
- ./data:/var/lib/garage/data
|
||||
@@ -0,0 +1,29 @@
|
||||
# One node on the Synology. Velero's kopia data is encrypted, so
|
||||
# compression would only cost CPU.
|
||||
#
|
||||
# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
|
||||
# shutdown, and a single node has no other node to repair from. With
|
||||
# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
|
||||
# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
|
||||
# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
|
||||
# with `garage convert-db`.
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
db_engine = "sqlite"
|
||||
metadata_fsync = true
|
||||
data_fsync = true
|
||||
metadata_auto_snapshot_interval = "6h"
|
||||
replication_factor = 1
|
||||
compression_level = "none"
|
||||
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "127.0.0.1:3901"
|
||||
rpc_secret = "@RPC_SECRET@"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
|
||||
[admin]
|
||||
api_bind_addr = "[::]:3903"
|
||||
admin_token = "@ADMIN_TOKEN@"
|
||||
@@ -19,6 +19,11 @@ let
|
||||
"serpbear.rubenhensen.nl"
|
||||
"git.rubenhensen.nl"
|
||||
"vic.rubenhensen.nl"
|
||||
# The group's DNS points these at rubenhensen.nl, i.e. here
|
||||
"scoutingmbg.nl"
|
||||
"www.scoutingmbg.nl"
|
||||
"foto.scoutingmbg.nl"
|
||||
"auth.scoutingmbg.nl"
|
||||
];
|
||||
|
||||
sniMapEntries =
|
||||
@@ -90,7 +95,7 @@ let
|
||||
conditions = [ "[CONNECTED] == true" "[CERTIFICATE_EXPIRATION] > 168h" ];
|
||||
}
|
||||
(gatusHttp "Scouting MBG" "scoutingmbg.nl" "https://scoutingmbg.nl" // gatusAlert)
|
||||
(gatusHttp "Scouting MBG" "foto.scoutingmbg.nl" "https://foto.scoutingmbg.nl/_app" // gatusAlert)
|
||||
(gatusHttp "Scouting MBG" "foto.scoutingmbg.nl" "https://foto.scoutingmbg.nl/api/server/ping" // gatusAlert)
|
||||
({
|
||||
name = "s2 (homelab)";
|
||||
group = "watchers";
|
||||
@@ -168,12 +173,18 @@ in
|
||||
# - mail.rubenhensen.nl (and anything else) → local stalwart on 8443
|
||||
# - tunneled hosts → home cluster ingress on 443
|
||||
# - s1 / ntfy1.rubenhensen.nl → local nginx on 8444 (Gatus, ntfy)
|
||||
#
|
||||
# Each connection starts with a PROXY header that carries the visitor's
|
||||
# address, so the home cluster sees who is asking instead of this box
|
||||
# (gateway/proxy-protocol.yaml in k8scd). nginx sends the header to
|
||||
# every upstream or none, so the local ones go through a hop on
|
||||
# 8445/8446 that takes it off again.
|
||||
streamConfig = ''
|
||||
map $ssl_preread_server_name $tunnel_upstream {
|
||||
${sniMapEntries}
|
||||
s1.rubenhensen.nl 127.0.0.1:8444;
|
||||
ntfy1.rubenhensen.nl 127.0.0.1:8444;
|
||||
default 127.0.0.1:8443;
|
||||
s1.rubenhensen.nl 127.0.0.1:8446;
|
||||
ntfy1.rubenhensen.nl 127.0.0.1:8446;
|
||||
default 127.0.0.1:8445;
|
||||
}
|
||||
|
||||
server {
|
||||
@@ -181,6 +192,17 @@ ${sniMapEntries}
|
||||
listen [::]:443;
|
||||
proxy_pass $tunnel_upstream;
|
||||
ssl_preread on;
|
||||
proxy_protocol on;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:8445 proxy_protocol;
|
||||
proxy_pass 127.0.0.1:8443;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:8446 proxy_protocol;
|
||||
proxy_pass 127.0.0.1:8444;
|
||||
}
|
||||
'';
|
||||
|
||||
|
||||
@@ -10,3 +10,4 @@ resources:
|
||||
- ntfy-service.yaml
|
||||
- certificate.yaml
|
||||
- httproute.yaml
|
||||
- networkpolicy.yaml
|
||||
@@ -0,0 +1,93 @@
|
||||
# ntfy faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces.
|
||||
# Push notifications go out over the internet.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 80
|
||||
# Gatus sends its alerts to ntfy in-cluster.
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: gatus
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: gatus
|
||||
ports:
|
||||
- port: 80
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: ntfy
|
||||
image: binwiederhier/ntfy:v2.28.0
|
||||
image: binwiederhier/ntfy:v2.34.0
|
||||
args: ["serve"]
|
||||
env:
|
||||
- name: NTFY_BASE_URL
|
||||
|
||||
+72
-23
@@ -1,11 +1,21 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:recommended"
|
||||
"config:recommended",
|
||||
"customManagers:githubActionsVersions"
|
||||
],
|
||||
"timezone": "Europe/Amsterdam",
|
||||
"argocd": {
|
||||
"description": "Every app folder; Renovate's regexes are RE2, which has no lookahead, so the other folders are skipped through ignorePaths",
|
||||
"managerFilePatterns": [
|
||||
"/apps/templates/.+\\.yaml$/"
|
||||
"/^[^/]+/.+\\.yaml$/"
|
||||
],
|
||||
"ignorePaths": [
|
||||
"nix-infra-machine/**",
|
||||
"backup-testing/**",
|
||||
"docs/**",
|
||||
"tools/**",
|
||||
".github/**"
|
||||
]
|
||||
},
|
||||
"helm-values": {
|
||||
@@ -14,26 +24,19 @@
|
||||
]
|
||||
},
|
||||
"kubernetes": {
|
||||
"description": "Every app folder, as for argocd",
|
||||
"managerFilePatterns": [
|
||||
"/apps/templates/.+\\.yaml$/",
|
||||
"/serpbear/.+\\.yaml$/",
|
||||
"/bolletjestrui/.+\\.yaml$/",
|
||||
"/gitea/.+\\.yaml$/",
|
||||
"/gatus/.+\\.yaml$/",
|
||||
"/ntfy/.+\\.yaml$/"
|
||||
"/^[^/]+/.+\\.yaml$/"
|
||||
],
|
||||
"ignorePaths": [
|
||||
"nix-infra-machine/**",
|
||||
"backup-testing/**",
|
||||
"docs/**",
|
||||
"tools/**",
|
||||
".github/**"
|
||||
]
|
||||
},
|
||||
"customManagers": [
|
||||
{
|
||||
"customType": "regex",
|
||||
"managerFilePatterns": [
|
||||
"/apps/templates/.+\\.yaml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"repoURL: https://github.com/(?<depName>[^/]+/[^/\\.]+)\\.git\\n\\s+targetRevision: (?<currentValue>.*)"
|
||||
],
|
||||
"datasourceTemplate": "github-tags"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"managerFilePatterns": [
|
||||
@@ -43,6 +46,17 @@
|
||||
"repoURL: (?<registryUrl>.*?)\\n\\s+targetRevision: (?<currentValue>[^\\s]+)\\n\\s+chart: (?<depName>[^\\s]+)"
|
||||
],
|
||||
"datasourceTemplate": "helm"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "CloudNativePG clusters name their image in imageName",
|
||||
"managerFilePatterns": [
|
||||
"/^[^/]+/postgresql-cluster[^/]*\\.yaml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"imageName: (?<depName>[^:\\s]+):(?<currentValue>\\S+)"
|
||||
],
|
||||
"datasourceTemplate": "docker"
|
||||
}
|
||||
],
|
||||
"packageRules": [
|
||||
@@ -55,12 +69,47 @@
|
||||
"separateMinorPatch": true
|
||||
},
|
||||
{
|
||||
"matchDatasources": [
|
||||
"github-tags"
|
||||
"description": "CI renders with the same Helm as Argo CD (3.5 ships Helm 4.2); move it together with Argo CD",
|
||||
"matchFileNames": [
|
||||
".github/workflows/**"
|
||||
],
|
||||
"semanticCommits": "enabled",
|
||||
"commitMessageTopic": "{{depName}}",
|
||||
"separateMinorPatch": true
|
||||
"matchDepNames": [
|
||||
"helm/helm"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "A PostgreSQL major upgrade needs pg_upgrade or a dump, not a tag bump",
|
||||
"matchDepNames": [
|
||||
"ghcr.io/cloudnative-pg/postgresql",
|
||||
"ghcr.io/tensorchord/cloudnative-vectorchord",
|
||||
"ghcr.io/tensorchord/cloudnative-pgvecto.rs"
|
||||
],
|
||||
"matchUpdateTypes": [
|
||||
"major"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "Patch updates of apps that hold little state and are easy to roll back, and of CI's tools, merge themselves once CI passes: three days after the release, in the evening or at the weekend, when the cluster can be reached. Not the databases, operators, Vault, authentik, Longhorn, Velero, immich, Gitea or WordPress. Renovate merges itself; the repo has no branch protection for GitHub's auto-merge to wait on",
|
||||
"matchFileNames": [
|
||||
"ntfy/**",
|
||||
"gatus/**",
|
||||
"freshrss/**",
|
||||
"serpbear/**",
|
||||
"actual-budget/**",
|
||||
".github/workflows/**"
|
||||
],
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
],
|
||||
"minimumReleaseAge": "3 days",
|
||||
"automerge": true,
|
||||
"platformAutomerge": false,
|
||||
"automergeSchedule": [
|
||||
"* 17-21 * * 1-5",
|
||||
"* 10-21 * * 0,6"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
# apiVersion: external-secrets.io/v1beta1
|
||||
# kind: ExternalSecret
|
||||
# metadata:
|
||||
# name: hensenio-smtp-secrets
|
||||
# namespace: blog
|
||||
# spec:
|
||||
# refreshInterval: "15m"
|
||||
# secretStoreRef:
|
||||
# name: vault-backend
|
||||
# kind: ClusterSecretStore
|
||||
# target:
|
||||
# name: hensenio-smtp
|
||||
# creationPolicy: Owner
|
||||
# data:
|
||||
# - secretKey: smtp-password
|
||||
# remoteRef:
|
||||
# key: kv/hensenio/smtp
|
||||
# property: password
|
||||
@@ -0,0 +1,82 @@
|
||||
# SerpBear faces the internet through Envoy, so its pods only reach each
|
||||
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||
# API or other namespaces. Its scraper APIs are on the internet.
|
||||
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||
# Same layout as mbgwp/networkpolicy.yaml.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 3000
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
@@ -56,6 +56,25 @@ spec:
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 160Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /login
|
||||
port: 3000
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 3000
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
volumeMounts:
|
||||
- mountPath: /app/data
|
||||
name: data
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render every Argo CD app in this repo the way Argo CD does and validate
|
||||
the output with kubeconform, so a broken manifest or a chart bump that no
|
||||
longer renders fails the PR instead of the sync.
|
||||
|
||||
Run from anywhere; needs helm, kustomize, kubeconform, git and PyYAML.
|
||||
The apps/ chart is rendered first, then every Application found in it and
|
||||
in the app folders. Sources:
|
||||
- a chart: helm template with the Application's values, value files and
|
||||
parameters
|
||||
- a path in this repo: helm template (Chart.yaml), kustomize build
|
||||
(kustomization.yaml) or the folder's YAML files (directory include and
|
||||
exclude)
|
||||
- a path in another git repository: cloned at targetRevision, then the
|
||||
same, with the Application's kustomize patches applied
|
||||
Every YAML file in the repo is also checked for duplicate keys, which
|
||||
YAML parsers resolve silently by keeping the last one.
|
||||
|
||||
Repo rules, on what this repo's own folders render (not on charts or
|
||||
remote bases):
|
||||
- no Secret with data: secrets come from Vault through ESO
|
||||
- every image has a tag other than latest
|
||||
- warnings (annotations on the PR) for containers without requests or
|
||||
probes
|
||||
"""
|
||||
import fnmatch
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
OWN_REPOS = {"https://github.com/rubenhensen/k8scd.git", "https://github.com/rubenhensen/k8scd"}
|
||||
SKIP_DIRS = {".git", ".github", "nix-infra-machine", "docs", "tools", "backup-testing"}
|
||||
# Helm's view of the cluster; the JSON schemas only go up to 1.34.
|
||||
KUBE_VERSION = "1.36.0"
|
||||
SCHEMA_VERSION = "1.34.12"
|
||||
CRD_SCHEMAS = ("https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/"
|
||||
"{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json")
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
# Registries rate-limit now and then (ghcr.io answers 429); try again.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
|
||||
except subprocess.CalledProcessError as e:
|
||||
transient = any(s in (e.stderr or "") for s in ("429", "toomanyrequests", "timeout", "connection reset"))
|
||||
if not transient or attempt == 2:
|
||||
raise
|
||||
time.sleep(10 * (attempt + 1))
|
||||
|
||||
|
||||
class UniqueKeyLoader(yaml.SafeLoader):
|
||||
pass
|
||||
|
||||
|
||||
def construct_mapping(loader, node, deep=False):
|
||||
keys = set()
|
||||
for key_node, _ in node.value:
|
||||
key = loader.construct_object(key_node, deep=deep)
|
||||
if key in keys:
|
||||
raise yaml.constructor.ConstructorError(
|
||||
None, None, f"duplicate key {key!r}", key_node.start_mark)
|
||||
keys.add(key)
|
||||
return loader.construct_mapping(node, deep)
|
||||
|
||||
|
||||
UniqueKeyLoader.add_constructor(
|
||||
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
|
||||
|
||||
|
||||
def yaml_files():
|
||||
for path in sorted(ROOT.rglob("*.y*ml")):
|
||||
rel = path.relative_to(ROOT)
|
||||
if rel.parts[0] in SKIP_DIRS or path.suffix not in (".yaml", ".yml"):
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def check_duplicate_keys():
|
||||
for path in yaml_files():
|
||||
if path.parent.name == "templates": # Helm templates aren't YAML yet
|
||||
continue
|
||||
try:
|
||||
list(yaml.load_all(path.read_text(), Loader=UniqueKeyLoader))
|
||||
except yaml.YAMLError as e:
|
||||
failures.append(f"{path.relative_to(ROOT)}: {e}")
|
||||
|
||||
|
||||
def pod_specs(doc):
|
||||
spec = doc.get("spec") or {}
|
||||
kind = doc.get("kind")
|
||||
if kind == "Pod":
|
||||
yield spec
|
||||
elif kind == "CronJob":
|
||||
yield spec.get("jobTemplate", {}).get("spec", {}).get("template", {}).get("spec", {})
|
||||
elif kind in ("Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"):
|
||||
yield spec.get("template", {}).get("spec", {})
|
||||
|
||||
|
||||
def check_rules(label, docs):
|
||||
for d in docs:
|
||||
name = f"{d.get('kind')}/{d.get('metadata', {}).get('name')}"
|
||||
if d.get("kind") == "Secret" and (d.get("data") or d.get("stringData")):
|
||||
failures.append(f"{label}: {name} has data in git; put it in Vault and use an ExternalSecret")
|
||||
for spec in pod_specs(d):
|
||||
containers = spec.get("containers", []) + spec.get("initContainers", [])
|
||||
for c in containers:
|
||||
image = c.get("image", "")
|
||||
tag = image.split("@")[0].rsplit("/", 1)[-1].partition(":")[2]
|
||||
if "@" not in image and tag in ("", "latest"):
|
||||
failures.append(f"{label}: {name} container {c.get('name')} has no fixed tag ({image})")
|
||||
for c in spec.get("containers", []):
|
||||
missing = []
|
||||
if not c.get("resources", {}).get("requests"):
|
||||
missing.append("requests")
|
||||
if d.get("kind") in ("Deployment", "StatefulSet", "DaemonSet") and not (
|
||||
c.get("readinessProbe") or c.get("livenessProbe")):
|
||||
missing.append("probes")
|
||||
if missing:
|
||||
print(f"::warning title={label}::{name} container {c.get('name')} has no {' or '.join(missing)}")
|
||||
|
||||
|
||||
def validate(label, manifests, rules=False):
|
||||
"""kubeconform on rendered manifests; returns the parsed documents."""
|
||||
try:
|
||||
docs = [d for d in yaml.safe_load_all(manifests) if d]
|
||||
except yaml.YAMLError:
|
||||
docs = None # kubeconform reports where
|
||||
if docs is not None:
|
||||
# Argo CD skips documents that aren't objects, like a values file
|
||||
# next to the manifests.
|
||||
objects = [d for d in docs if isinstance(d, dict) and "kind" in d]
|
||||
if len(objects) < len(docs):
|
||||
print(f" {label}: skipping {len(docs) - len(objects)} document(s) without a kind")
|
||||
manifests = yaml.safe_dump_all(objects, sort_keys=False)
|
||||
result = subprocess.run(
|
||||
["kubeconform", "-summary", "-output", "text", "-strict",
|
||||
"-kubernetes-version", SCHEMA_VERSION, "-ignore-missing-schemas",
|
||||
"-schema-location", "default", "-schema-location", CRD_SCHEMAS],
|
||||
input=manifests, capture_output=True, text=True)
|
||||
lines = [l for l in result.stdout.splitlines() if l.strip()]
|
||||
summary = lines[-1] if lines else ""
|
||||
print(f" {label}: {summary.replace('Summary: ', '')}")
|
||||
if result.returncode != 0:
|
||||
for l in lines[:-1]:
|
||||
print(f" {l}")
|
||||
failures.append(f"{label}: {summary or result.stderr.strip()}")
|
||||
if rules and docs:
|
||||
check_rules(label, [d for d in docs if isinstance(d, dict)])
|
||||
return docs or []
|
||||
|
||||
|
||||
def helm_template(name, namespace, chart_args, helm):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
args = ["helm", "template", helm.get("releaseName", name), *chart_args,
|
||||
"--namespace", namespace, "--kube-version", KUBE_VERSION,
|
||||
"--include-crds"]
|
||||
if helm.get("values"):
|
||||
values = pathlib.Path(tmp, "values.yaml")
|
||||
values.write_text(helm["values"])
|
||||
args += ["-f", str(values)]
|
||||
if helm.get("valuesObject"):
|
||||
values = pathlib.Path(tmp, "values-object.yaml")
|
||||
values.write_text(yaml.safe_dump(helm["valuesObject"]))
|
||||
args += ["-f", str(values)]
|
||||
for vf in helm.get("valueFiles", []):
|
||||
# $values/<path> points at this repo through a source with ref: values
|
||||
args += ["-f", str(ROOT / vf.split("/", 1)[1]) if vf.startswith("$") else vf]
|
||||
for p in helm.get("parameters", []):
|
||||
flag = "--set-string" if p.get("forceString") else "--set"
|
||||
args += [flag, f"{p['name']}={p['value']}"]
|
||||
return run(args)
|
||||
|
||||
|
||||
def directory(path, spec):
|
||||
include = spec.get("include", "*.yaml")
|
||||
exclude = spec.get("exclude", "")
|
||||
files = sorted(path.rglob("*") if spec.get("recurse") else path.glob("*"))
|
||||
docs = []
|
||||
for f in files:
|
||||
if f.suffix not in (".yaml", ".yml", ".json") or not f.is_file():
|
||||
continue
|
||||
name = f.name
|
||||
if not any(fnmatch.fnmatch(name, g) for g in include.strip("{}").split(",")):
|
||||
continue
|
||||
if exclude and any(fnmatch.fnmatch(name, g) for g in exclude.strip("{}").split(",")):
|
||||
continue
|
||||
docs.append(f.read_text())
|
||||
return "\n---\n".join(docs)
|
||||
|
||||
|
||||
def kustomize(path, spec):
|
||||
if not spec:
|
||||
return run(["kustomize", "build", str(path)])
|
||||
# The Application's kustomize options on top, as Argo CD does.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
k = {"apiVersion": "kustomize.config.k8s.io/v1beta1", "kind": "Kustomization",
|
||||
"resources": [os.path.relpath(path, tmp)]}
|
||||
for key in ("patches", "images", "namespace", "commonLabels", "namePrefix", "nameSuffix"):
|
||||
if key in spec:
|
||||
k[key] = spec[key]
|
||||
pathlib.Path(tmp, "kustomization.yaml").write_text(yaml.safe_dump(k))
|
||||
return run(["kustomize", "build", "--load-restrictor", "LoadRestrictionsNone", tmp])
|
||||
|
||||
|
||||
def render_path(app, src, base):
|
||||
path = base / src.get("path", ".")
|
||||
namespace = app["spec"]["destination"].get("namespace", "default")
|
||||
if (path / "Chart.yaml").exists():
|
||||
if (path / "Chart.lock").exists():
|
||||
run(["helm", "dependency", "build", str(path)])
|
||||
return helm_template(app["metadata"]["name"], namespace, [str(path)], src.get("helm", {}))
|
||||
if (path / "kustomization.yaml").exists() or src.get("kustomize"):
|
||||
return kustomize(path, src.get("kustomize"))
|
||||
return directory(path, src.get("directory", {}))
|
||||
|
||||
|
||||
def render(app, src, clones):
|
||||
name = app["metadata"]["name"]
|
||||
namespace = app["spec"]["destination"].get("namespace", "default")
|
||||
repo = src["repoURL"]
|
||||
if src.get("chart"):
|
||||
if repo.startswith("http"):
|
||||
chart_args = [src["chart"], "--repo", repo]
|
||||
else:
|
||||
chart_args = [f"oci://{repo.rstrip('/')}/{src['chart']}"]
|
||||
chart_args += ["--version", src["targetRevision"]]
|
||||
return helm_template(name, namespace, chart_args, src.get("helm", {}))
|
||||
if src.get("ref") and "path" not in src:
|
||||
return None # only provides value files
|
||||
if repo in OWN_REPOS:
|
||||
return render_path(app, src, ROOT)
|
||||
key = (repo, src.get("targetRevision", "HEAD"))
|
||||
if key not in clones:
|
||||
tmp = tempfile.mkdtemp()
|
||||
run(["git", "clone", "--quiet", "--depth", "1", "--branch", key[1], repo, tmp])
|
||||
clones[key] = pathlib.Path(tmp)
|
||||
return render_path(app, src, clones[key])
|
||||
|
||||
|
||||
def applications(docs):
|
||||
return [d for d in docs if isinstance(d, dict) and d.get("kind") == "Application"
|
||||
and d.get("apiVersion", "").startswith("argoproj.io/")]
|
||||
|
||||
|
||||
def main():
|
||||
print("Duplicate keys")
|
||||
check_duplicate_keys()
|
||||
|
||||
print("Applications")
|
||||
apps = applications(validate("apps (chart)", run(
|
||||
["helm", "template", "apps", str(ROOT / "apps"), "--kube-version", KUBE_VERSION])))
|
||||
for path in yaml_files():
|
||||
if path.parts[len(ROOT.parts)] == "apps":
|
||||
continue
|
||||
try:
|
||||
apps += applications([d for d in yaml.safe_load_all(path.read_text()) if d])
|
||||
except yaml.YAMLError:
|
||||
pass # reported by the duplicate key check or by the app that uses it
|
||||
|
||||
clones = {}
|
||||
seen = set()
|
||||
for app in sorted(apps, key=lambda a: a["metadata"]["name"]):
|
||||
name = app["metadata"]["name"]
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
spec = app["spec"]
|
||||
sources = spec.get("sources") or [spec["source"]]
|
||||
for i, src in enumerate(sources):
|
||||
label = name if len(sources) == 1 else f"{name}[{i}]"
|
||||
try:
|
||||
manifests = render(app, src, clones)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f" {label}: render failed")
|
||||
lines = []
|
||||
for line in (e.stderr or e.stdout).strip().splitlines():
|
||||
if not lines or line != lines[-1]: # Helm repeats schema errors
|
||||
lines.append(line)
|
||||
for line in lines[:20]:
|
||||
print(f" {line}")
|
||||
failures.append(f"{label}: render failed")
|
||||
continue
|
||||
if manifests is not None:
|
||||
own = src.get("repoURL") in OWN_REPOS and not src.get("chart") and not (
|
||||
ROOT / src.get("path", ".") / "Chart.yaml").exists()
|
||||
validate(label, manifests, rules=own)
|
||||
|
||||
if failures:
|
||||
print(f"\n{len(failures)} problem(s):")
|
||||
for f in failures:
|
||||
print(f"- {f}")
|
||||
sys.exit(1)
|
||||
print("\nAll apps render and validate.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,7 +3,7 @@ module dqlite-vacuum
|
||||
go 1.27.3
|
||||
|
||||
require (
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1
|
||||
github.com/canonical/go-dqlite/v3 v3.0.4
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
|
||||
@@ -2,7 +2,10 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
|
||||
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -26,7 +29,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# Restart the Vault pods one at a time and unseal each one: the standbys
|
||||
# first, the active node last, as HashiCorp's HA upgrade guide asks. The
|
||||
# StatefulSet uses OnDelete, so this is how a new image from
|
||||
# vault/overrides.yaml reaches the pods once Argo CD has synced it. Also
|
||||
# useful after a reboot, minus the restarts: see "unseal only" below.
|
||||
#
|
||||
# Run it on a machine with kubectl access and the unseal key at hand:
|
||||
# tools/vault-roll.sh 2.2.1
|
||||
# It asks for the key once per pod (hidden input) and stores nothing.
|
||||
# Stop at any point with Ctrl-C; pods already done stay on the new version.
|
||||
#
|
||||
# Unseal only (after a reboot, all three come up sealed):
|
||||
# for p in vault-0 vault-1 vault-2; do kubectl -n vault exec -ti $p -- vault operator unseal; done
|
||||
set -euo pipefail
|
||||
ns=vault
|
||||
target=${1:?usage: $0 <version the pods should run, e.g. 2.2.1>}
|
||||
|
||||
image=$(kubectl -n $ns get sts vault -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||
if [[ $image != *:$target ]]; then
|
||||
echo "The StatefulSet has $image, not $target. Sync the vault app first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pods() { kubectl -n $ns get pods -l "app.kubernetes.io/name=vault,component=server,vault-active=$1" -o name | sed 's|pod/||'; }
|
||||
status() { kubectl -n $ns exec "$1" -- vault status -format=json 2>/dev/null || true; }
|
||||
summary() {
|
||||
status "$1" | jq -r --arg p "$1" '"\($p): \(.version), sealed \(.sealed), \(if .is_self then "active" else "standby" end), raft index \(.raft_committed_index)"'
|
||||
}
|
||||
|
||||
roll() {
|
||||
local pod=$1 rc
|
||||
echo
|
||||
echo "== $pod"
|
||||
kubectl -n $ns delete pod "$pod" --wait=true
|
||||
# vault status exits 0 when unsealed, 2 when sealed, 1 on errors (and
|
||||
# kubectl exec exits 1 while the container isn't running yet).
|
||||
while :; do
|
||||
rc=0
|
||||
kubectl -n $ns exec "$pod" -- vault status >/dev/null 2>&1 || rc=$?
|
||||
[[ $rc -ne 1 ]] && break
|
||||
sleep 2
|
||||
done
|
||||
if [[ $rc -eq 2 ]]; then
|
||||
kubectl -n $ns exec -ti "$pod" -- vault operator unseal
|
||||
fi
|
||||
kubectl -n $ns wait --for=condition=Ready "pod/$pod" --timeout=180s
|
||||
summary "$pod"
|
||||
local version
|
||||
version=$(status "$pod" | jq -r .version)
|
||||
if [[ $version != "$target" ]]; then
|
||||
echo "$pod runs $version, expected $target. Stopping." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
active=$(pods true)
|
||||
standbys=$(pods false)
|
||||
if [[ $(echo "$active" | wc -w) -ne 1 || $(echo "$standbys" | wc -w) -ne 2 ]]; then
|
||||
echo "Expected one active node and two standbys, found active: $active, standbys: $standbys" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Active: $active. Standbys: $(echo $standbys)."
|
||||
|
||||
for pod in $standbys; do
|
||||
roll "$pod"
|
||||
done
|
||||
# The active node goes last. Deleting the pod shuts Vault down cleanly, so
|
||||
# it hands the HA lock to an upgraded standby right away; no step-down.
|
||||
roll "$active"
|
||||
|
||||
echo
|
||||
echo "== done"
|
||||
for pod in vault-0 vault-1 vault-2; do summary "$pod"; done
|
||||
@@ -33,7 +33,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: configure
|
||||
image: hashicorp/vault:1.15
|
||||
image: hashicorp/vault:2.2.1
|
||||
env:
|
||||
- name: VAULT_ADDR
|
||||
value: "http://vault-active.vault.svc:8200"
|
||||
@@ -51,7 +51,7 @@ spec:
|
||||
args:
|
||||
- |
|
||||
vault auth enable oidc 2>/dev/null || true
|
||||
vault auth tune -listing_visibility=unauth oidc/
|
||||
vault auth tune -listing-visibility=unauth oidc/
|
||||
vault policy write admin - <<'POLICY'
|
||||
path "*" {
|
||||
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
# Vault holds every secret, so only ESO, the UI through Envoy, and the
|
||||
# node reach it (the API server calls the injector's webhook from there;
|
||||
# 10.1.7.64 is the node's Calico VXLAN address,
|
||||
# projectcalico.org/IPv4VXLANTunnelAddr). Nothing uses agent injection.
|
||||
# The pods reach each other (raft on 8201, the OIDC job), DNS, the API
|
||||
# server (service registration labels the active pod, the injector) and
|
||||
# the internet (the OIDC login through authentik's public address).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: external-secrets
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-secrets
|
||||
ports:
|
||||
- port: 8200
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||
# hung).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: acme-http01-solver
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
acme.cert-manager.io/http01-solver: "true"
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: envoy
|
||||
ports:
|
||||
- port: 8089
|
||||
+10
-2
@@ -3,6 +3,12 @@ global:
|
||||
tlsDisable: true
|
||||
injector:
|
||||
enabled: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
# The chart's default anti-affinity forbids two injector pods on one
|
||||
# node, so on this single node a rolling update can never start the new
|
||||
# pod while the old one runs. Recreate removes the old pod first, which
|
||||
@@ -14,9 +20,11 @@ injector:
|
||||
server:
|
||||
# Pinned to the running version: the chart would otherwise pull its own
|
||||
# default on the next pod restart, and with Shamir unsealing a Vault
|
||||
# upgrade needs someone with the unseal keys at hand.
|
||||
# upgrade needs someone with the unseal keys at hand. The StatefulSet
|
||||
# uses OnDelete, so after a sync tools/vault-roll.sh <tag> restarts and
|
||||
# unseals the pods one at a time.
|
||||
image:
|
||||
tag: "1.17.2"
|
||||
tag: "2.2.1"
|
||||
dataStorage:
|
||||
storageClass: longhorn
|
||||
standalone:
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
|
||||
# AWS credentials file format the aws plugin and kopia read.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: velero-s3-credentials
|
||||
spec:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
kind: ClusterSecretStore
|
||||
refreshInterval: 15m
|
||||
target:
|
||||
name: velero-s3-credentials
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
engineVersion: v2
|
||||
data:
|
||||
cloud: |
|
||||
[default]
|
||||
aws_access_key_id={{ .access_key_id }}
|
||||
aws_secret_access_key={{ .secret_access_key }}
|
||||
data:
|
||||
- secretKey: access_key_id
|
||||
remoteRef:
|
||||
key: kv/velero-s3
|
||||
property: access_key_id
|
||||
- secretKey: secret_access_key
|
||||
remoteRef:
|
||||
key: kv/velero-s3
|
||||
property: secret_access_key
|
||||
@@ -1,23 +0,0 @@
|
||||
# apiVersion: external-secrets.io/v1beta1
|
||||
# kind: ExternalSecret
|
||||
# metadata:
|
||||
# name: velero-minio-credentials
|
||||
# namespace: velero
|
||||
# spec:
|
||||
# refreshInterval: "15s"
|
||||
# secretStoreRef:
|
||||
# name: vault-backend
|
||||
# kind: ClusterSecretStore
|
||||
# target:
|
||||
# name: velero-minio-credentials
|
||||
# creationPolicy: Owner
|
||||
# data:
|
||||
# - secretKey: cloud
|
||||
# remoteRef:
|
||||
# key: velero/minio-credentials
|
||||
# template:
|
||||
# engine: gotmpl
|
||||
# data: |
|
||||
# [default]
|
||||
# aws_access_key_id={{ .aws_access_key_id }}
|
||||
# aws_secret_access_key={{ .aws_secret_access_key }}
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: nas-minio-external
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: 192.168.1.99
|
||||
ports:
|
||||
- port: 9000
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,52 @@
|
||||
# Velero, its node agent, and the pods it starts here (the data movers of
|
||||
# each pod volume backup or restore, kopia's maintenance jobs, the chart's
|
||||
# CRD upgrade job) reach each other, DNS, the API server (the node, port
|
||||
# 16443) and Garage on the NAS (192.168.1.99:3900, nas/garage). The node
|
||||
# agent reads volumes through its hostPath mount, not the network. Besides
|
||||
# their own namespace they take connections only from the node (kubelet
|
||||
# probes; 10.1.7.64 is the node's Calico VXLAN address,
|
||||
# projectcalico.org/IPv4VXLANTunnelAddr); nothing scrapes the metrics on
|
||||
# 8085. MinIO (the NAS, port 9000) is not allowed: going back to it means
|
||||
# adding its port here as well.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector: {}
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
- ipBlock:
|
||||
cidr: 10.1.7.64/32
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.204/32
|
||||
ports:
|
||||
- port: 16443
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.1.99/32
|
||||
ports:
|
||||
- port: 3900
|
||||
@@ -1,6 +1,8 @@
|
||||
# Volume policy for the backup schedules. emptyDir volumes hold nothing
|
||||
# worth keeping, and backing them up fails when the pod is gone before its
|
||||
# volume is read, as happens with Velero's own repository maintenance jobs.
|
||||
# PVCs in Longhorn's no-backup group (caches, such as immich's ML models)
|
||||
# are skipped too.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
@@ -14,3 +16,8 @@ data:
|
||||
- emptyDir
|
||||
action:
|
||||
type: skip
|
||||
- conditions:
|
||||
pvcLabels:
|
||||
recurring-job-group.longhorn.io/no-backup: enabled
|
||||
action:
|
||||
type: skip
|
||||
+30
-9
@@ -27,31 +27,52 @@ spec:
|
||||
- mountPath: /target
|
||||
name: plugins
|
||||
configuration:
|
||||
# Garage on the NAS (nas/garage). Until 2026-10-26 this was MinIO on
|
||||
# the same NAS; the bucket was copied over with rclone, so the
|
||||
# existing backups and kopia repositories carried on.
|
||||
backupStorageLocation:
|
||||
- name: default
|
||||
bucket: velero-backups
|
||||
provider: aws
|
||||
config:
|
||||
region: minio
|
||||
region: garage
|
||||
s3ForcePathStyle: true
|
||||
s3Url: http://192.168.1.99:9000
|
||||
insecureSkipTLSVerify: "true"
|
||||
s3Url: http://192.168.1.99:3900
|
||||
volumeSnapshotLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
config:
|
||||
region: minio
|
||||
region: garage
|
||||
features: EnableCSI
|
||||
# Kopia maintenance ran every hour for each of the 15 repositories,
|
||||
# and once a day as a full run of several minutes that writes its
|
||||
# cache to the node's only SSD, which stalls dqlite and makes
|
||||
# controllers lose their leases (docs/upgrade-k8s-1.33.md). Daily
|
||||
# and throttled instead. Existing BackupRepositories keep their own
|
||||
# maintenanceFrequency; they were patched to 24h by hand.
|
||||
defaultRepoMaintainFrequency: 24h
|
||||
repositoryMaintenanceJob:
|
||||
repositoryConfigData:
|
||||
global:
|
||||
keepLatestMaintenanceJobs: 3
|
||||
podResources:
|
||||
cpuRequest: 100m
|
||||
cpuLimit: 500m
|
||||
memoryRequest: 256Mi
|
||||
memoryLimit: 2Gi
|
||||
credentials:
|
||||
# Use existing secret created by External Secrets Operator
|
||||
# existingSecret: velero-minio-credentials
|
||||
# extraEnvVars:
|
||||
# AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
|
||||
existingSecret: cloud-credentials
|
||||
# From Vault, velero/external-secret.yaml
|
||||
existingSecret: velero-s3-credentials
|
||||
deployNodeAgent: true
|
||||
nodeAgent:
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
# 512Mi got the server OOMKilled 314 times, including halfway
|
||||
# through a backup of ~14.7k items.
|
||||
resources:
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
- staging env
|
||||
- automated backup testing
|
||||
- conftest policy as code voor clean gitops repo regels. Leg fouten vast zodat ze niet meer gebeuren.
|
||||
- beter begrip van Longhorn en deletion policies enzo.
|
||||
- minder handmatig werk als ik een app wil deployen?
|
||||
- 3-node cluster
|
||||
- CI/CD pipeline
|
||||
- image registry
|
||||
- centralized logging met loki en grafana
|
||||
- metrics met prometheus en grafana
|
||||
- alertmanager met goede alerts
|
||||
- uitzoeken wat goede alerts zijn
|
||||
- hardware management met behulp van die idempotem client ding, ben de naam kwijt.
|
||||
Dan een nieuwe server opzetten compleet met dat,
|
||||
en dan overgaan door het als een nieuwe node toe te voegen en de andere uit te zetten.
|
||||
- authentik en alles met sso openid verbinden
|
||||
- Renovate voor automatische updates zodat ik niet zo achter raak.
|
||||
- Trivy vulnarability scanning (CVE scans)
|
||||
- Network policies
|
||||
- clean up repo
|
||||
- remove hardcodes secrets.
|
||||
- health probes to all services
|
||||
- limits and requests for all services.
|
||||
- non root security policy for all services.
|
||||
- kube hunter network scans
|
||||
- Draai mijn eigen github alternative
|
||||
- Scale down pvc immich to 200gb.
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user