mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-15 22:29:09 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b29616990c | ||
|
|
9f946d5e68 | ||
|
|
f08737c221 | ||
|
|
65858d3f0b | ||
|
|
b6be49b701 | ||
|
|
d6779a399f | ||
|
|
e042fdd8de | ||
|
|
e8145e12d7 | ||
|
|
3d1b7dd230 | ||
|
|
aaae182844 | ||
|
|
e0e64e1097 | ||
|
|
b4e992d092 | ||
|
|
6ee7fd6f5e | ||
|
|
7db73a100e | ||
|
|
40cd825602 | ||
|
|
bbb56e0e25 | ||
|
|
a4b09fc51e | ||
|
|
0589bb3e08 | ||
|
|
c298ad8746 | ||
|
|
fe105a4fa7 | ||
|
|
1afd4b6bbd | ||
|
|
b4663216db | ||
|
|
955c22c665 | ||
|
|
f7396b3cb7 | ||
|
|
1f894996e6 | ||
|
|
c380bfc0cf | ||
|
|
c3597ef0a6 | ||
|
|
baa8c70b55 | ||
|
|
1efa6c2b93 | ||
|
|
15a92294a6 | ||
|
|
144b9dda2b | ||
|
|
920e54b43c | ||
|
|
665147a691 | ||
|
|
1ba1d5f1a0 | ||
|
|
285a64f980 | ||
|
|
6cf41b0f14 | ||
|
|
5678adfa82 | ||
|
|
2e2d367f63 | ||
|
|
2a1394763d | ||
|
|
86bbf25145 | ||
|
|
92d5147994 | ||
|
|
9e88a92ff7 | ||
|
|
7b12ee2028 | ||
|
|
039b956764 | ||
|
|
d187f55e57 | ||
|
|
a36fc6f348 | ||
|
|
66207de2dd | ||
|
|
a8ccbef274 | ||
|
|
d0ce5ddbec | ||
|
|
c384ae921f | ||
|
|
13b92fff54 | ||
|
|
3834f7a354 | ||
|
|
8110a235a9 | ||
|
|
fa33bf44ae | ||
|
|
d4e99d28c7 | ||
|
|
c8a81e805d | ||
|
|
3fd585de5f | ||
|
|
6c727346f6 | ||
|
|
ed828b426f | ||
|
|
0dc69d34f0 | ||
|
|
30540ea579 | ||
|
|
53d74768ad | ||
|
|
b6462fb3a9 | ||
|
|
bed68c70a8 | ||
|
|
3991a8d726 | ||
|
|
f6186d5a11 | ||
|
|
2ccaea24a8 | ||
|
|
a2abb66fea | ||
|
|
b721c80599 | ||
|
|
09026a5ff0 | ||
|
|
1fcfd3ad61 | ||
|
|
648f99c505 | ||
|
|
20bd3a4bca | ||
|
|
6cba1a9363 | ||
|
|
acfce6645e | ||
|
|
0de86dd65c | ||
|
|
d7ffdc05f6 | ||
|
|
c0f8034e14 | ||
|
|
a15a4f7c75 | ||
|
|
bdafcaf1e0 | ||
|
|
03c3196242 | ||
|
|
b072099cc1 | ||
|
|
e10ca337f1 | ||
|
|
c72b08867f | ||
|
|
d85362ce3b | ||
|
|
018f528fb3 | ||
|
|
714712b862 | ||
|
|
7dc13466cb | ||
|
|
9b77596bcb | ||
|
|
fbf2cb2d92 | ||
|
|
122873ca5c | ||
|
|
71449dd185 | ||
|
|
297d2dffa5 | ||
|
|
46117ee658 | ||
|
|
2a5bbc7f44 | ||
|
|
d7b0372f1a | ||
|
|
0ad6ca816c | ||
|
|
3e252799a4 | ||
|
|
bfd5619e9e | ||
|
|
0b38d8dec8 | ||
|
|
083353fb9d | ||
|
|
23f9c5c60a | ||
|
|
a8bf3d5b64 | ||
|
|
618954ae2c | ||
|
|
44dc85d306 | ||
|
|
b60735f6b0 | ||
|
|
3a57e5be48 | ||
|
|
1611f5d1ef | ||
|
|
68b692a112 | ||
|
|
6b2f5855d7 | ||
|
|
c25b8883da | ||
|
|
18f9e2efe6 | ||
|
|
5f5f5b01f0 | ||
|
|
6206ab5a74 | ||
|
|
075b3686c3 | ||
|
|
45d1ea0d93 |
No files matched your search
@@ -18,7 +18,7 @@ concurrency:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
validate:
|
validate:
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-26.04
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://community-charts.github.io/helm-charts
|
repoURL: https://community-charts.github.io/helm-charts
|
||||||
targetRevision: 1.8.9
|
targetRevision: 1.9.5
|
||||||
chart: actualbudget
|
chart: actualbudget
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
|
|||||||
@@ -55,3 +55,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -103,6 +103,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 70m
|
cpu: 70m
|
||||||
memory: 560Mi
|
memory: 560Mi
|
||||||
|
limits:
|
||||||
|
memory: 1536Mi
|
||||||
volumes:
|
volumes:
|
||||||
- name: patched-ca-bundle
|
- name: patched-ca-bundle
|
||||||
configMap:
|
configMap:
|
||||||
@@ -192,6 +194,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: 100m
|
||||||
memory: 336Mi
|
memory: 336Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
volumes:
|
volumes:
|
||||||
- name: patched-ca-bundle
|
- name: patched-ca-bundle
|
||||||
configMap:
|
configMap:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://charts.jetstack.io
|
repoURL: https://charts.jetstack.io
|
||||||
targetRevision: v1.21.2
|
targetRevision: v1.21.3
|
||||||
chart: cert-manager
|
chart: cert-manager
|
||||||
helm:
|
helm:
|
||||||
parameters:
|
parameters:
|
||||||
@@ -38,16 +38,22 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 48Mi
|
memory: 48Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
cainjector:
|
cainjector:
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 80Mi
|
memory: 80Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
webhook:
|
webhook:
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 32Mi
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
syncPolicy:
|
syncPolicy:
|
||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: cloudnative-pg-folder
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
|
targetRevision: HEAD
|
||||||
|
path: cloudnative-pg
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: cnpg-system
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
selfHeal: true
|
||||||
@@ -28,6 +28,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 48Mi
|
memory: 48Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: cnpg-system
|
namespace: cnpg-system
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: default-folder
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
|
targetRevision: HEAD
|
||||||
|
path: default
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: default
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
selfHeal: true
|
||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
repoURL: docker.io/envoyproxy
|
repoURL: docker.io/envoyproxy
|
||||||
chart: gateway-helm
|
chart: gateway-helm
|
||||||
# v1.9 supports Kubernetes 1.33 to 1.36
|
# v1.9 supports Kubernetes 1.33 to 1.36
|
||||||
targetRevision: v1.9.2
|
targetRevision: 1.9.3
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
config:
|
config:
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: external-secrets-folder
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
|
targetRevision: HEAD
|
||||||
|
path: external-secrets
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: external-secrets
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
selfHeal: true
|
||||||
@@ -17,23 +17,29 @@ spec:
|
|||||||
sources:
|
sources:
|
||||||
- chart: external-secrets
|
- chart: external-secrets
|
||||||
repoURL: https://charts.external-secrets.io
|
repoURL: https://charts.external-secrets.io
|
||||||
targetRevision: 2.13.0
|
targetRevision: 2.14.0
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 20m
|
cpu: 20m
|
||||||
memory: 64Mi
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
certController:
|
certController:
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 64Mi
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
webhook:
|
webhook:
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 32Mi
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: external-secrets
|
namespace: external-secrets
|
||||||
@@ -12,7 +12,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://groundhog2k.github.io/helm-charts/
|
repoURL: https://groundhog2k.github.io/helm-charts/
|
||||||
targetRevision: 0.15.0
|
targetRevision: 0.16.8
|
||||||
chart: wordpress
|
chart: wordpress
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
@@ -101,6 +101,23 @@ spec:
|
|||||||
timeoutSeconds: 5
|
timeoutSeconds: 5
|
||||||
failureThreshold: 30
|
failureThreshold: 30
|
||||||
|
|
||||||
|
# apache2.conf includes ports.conf after mods-enabled/*.conf, so the
|
||||||
|
# prefork settings here replace mpm_prefork.conf's MaxRequestWorkers
|
||||||
|
# 150. A worker takes ~45-60 MB (PSS), so 150 never fit in 1Gi: a
|
||||||
|
# crawler asking for 88 pages at once got the container OOMKilled on
|
||||||
|
# 2026-10-25. With 12 workers a burst waits in the listen backlog
|
||||||
|
# instead. Listen is the chart's default (containerPort).
|
||||||
|
apachePortsConfig: |
|
||||||
|
Listen 8000
|
||||||
|
<IfModule mpm_prefork_module>
|
||||||
|
StartServers 4
|
||||||
|
MinSpareServers 2
|
||||||
|
MaxSpareServers 6
|
||||||
|
ServerLimit 12
|
||||||
|
MaxRequestWorkers 12
|
||||||
|
MaxConnectionsPerChild 1000
|
||||||
|
</IfModule>
|
||||||
|
|
||||||
# Custom PHP configuration
|
# Custom PHP configuration
|
||||||
customPhpConfig: |
|
customPhpConfig: |
|
||||||
upload_max_filesize = 64M
|
upload_max_filesize = 64M
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: plugin-barman-cloud
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
chart: plugin-barman-cloud
|
||||||
|
repoURL: https://cloudnative-pg.github.io/charts
|
||||||
|
targetRevision: 0.8.1
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
# CNPG's barman cloud plugin, which backs the PostgreSQL clusters
|
||||||
|
# up to Garage on the NAS (authentik/postgresql-backup.yaml,
|
||||||
|
# immich/postgresql-backup.yaml). It has to run in the operator's
|
||||||
|
# namespace; the chart gets its TLS certificates from cert-manager.
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: cnpg-system
|
||||||
|
syncPolicy:
|
||||||
|
syncOptions:
|
||||||
|
- ServerSideApply=true
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
@@ -7,7 +7,7 @@ spec:
|
|||||||
project: default
|
project: default
|
||||||
source:
|
source:
|
||||||
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
||||||
targetRevision: v8.2.1
|
targetRevision: v8.7.0
|
||||||
path: deploy/kubernetes/snapshot-controller
|
path: deploy/kubernetes/snapshot-controller
|
||||||
kustomize:
|
kustomize:
|
||||||
patches:
|
patches:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Requests from what the components used on 2026-10-25 (kubectl top every
|
# Requests from what the components used on 2026-10-25 (kubectl top every
|
||||||
# 5 minutes; highest value plus 10%). No limits until there are daytime
|
# 5 minutes; highest value plus 10%). Memory limits at about twice the
|
||||||
# numbers, see docs/roadmap.md.
|
# highest value seen then or on 2026-10-27, rounded up; no CPU limits.
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: StatefulSet
|
kind: StatefulSet
|
||||||
metadata:
|
metadata:
|
||||||
@@ -14,6 +14,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 150m
|
cpu: 150m
|
||||||
memory: 704Mi
|
memory: 704Mi
|
||||||
|
limits:
|
||||||
|
memory: 2560Mi
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
@@ -28,6 +30,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: 100m
|
||||||
memory: 128Mi
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
@@ -42,6 +46,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 64Mi
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
@@ -56,6 +62,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 20m
|
cpu: 20m
|
||||||
memory: 96Mi
|
memory: 96Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
@@ -70,6 +78,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 48Mi
|
memory: 48Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
@@ -84,6 +94,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 48Mi
|
memory: 48Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
@@ -98,3 +110,5 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 32Mi
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
@@ -108,7 +108,8 @@ spec:
|
|||||||
---
|
---
|
||||||
# The CNPG operator checks each instance on its status port and connects to
|
# The CNPG operator checks each instance on its status port and connects to
|
||||||
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
||||||
# through the API server.
|
# through the API server. The barman cloud sidecar sends the WAL and the
|
||||||
|
# base backups to Garage on the NAS (postgresql-backup.yaml).
|
||||||
apiVersion: networking.k8s.io/v1
|
apiVersion: networking.k8s.io/v1
|
||||||
kind: NetworkPolicy
|
kind: NetworkPolicy
|
||||||
metadata:
|
metadata:
|
||||||
@@ -135,3 +136,34 @@ spec:
|
|||||||
cidr: 192.168.1.204/32
|
cidr: 192.168.1.204/32
|
||||||
ports:
|
ports:
|
||||||
- port: 16443
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.99/32
|
||||||
|
ports:
|
||||||
|
- port: 3900
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
# Backups of authentik's database by CNPG itself, through the barman cloud
|
||||||
|
# plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base backup every
|
||||||
|
# night, and every WAL segment as soon as it is full or 5 minutes old
|
||||||
|
# (archive_timeout). Both go to the bucket cnpg-authentik in Garage on the
|
||||||
|
# NAS (nas/garage), so the database can be restored to any moment of the
|
||||||
|
# last 30 days. Velero copies the data directory while PostgreSQL runs,
|
||||||
|
# which is not a consistent copy.
|
||||||
|
# The key is Garage's cnpg-authentik, in Vault as kv/cnpg-authentik-s3; it
|
||||||
|
# can only reach this bucket.
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: authentik-postgres-s3
|
||||||
|
spec:
|
||||||
|
secretStoreRef:
|
||||||
|
name: vault-backend
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
refreshInterval: 15m
|
||||||
|
target:
|
||||||
|
name: authentik-postgres-s3
|
||||||
|
creationPolicy: Owner
|
||||||
|
template:
|
||||||
|
engineVersion: v2
|
||||||
|
data:
|
||||||
|
ACCESS_KEY_ID: "{{ .access_key_id }}"
|
||||||
|
ACCESS_SECRET_KEY: "{{ .secret_access_key }}"
|
||||||
|
# Garage rejects requests signed for another region
|
||||||
|
REGION: garage
|
||||||
|
data:
|
||||||
|
- secretKey: access_key_id
|
||||||
|
remoteRef:
|
||||||
|
key: kv/cnpg-authentik-s3
|
||||||
|
property: access_key_id
|
||||||
|
- secretKey: secret_access_key
|
||||||
|
remoteRef:
|
||||||
|
key: kv/cnpg-authentik-s3
|
||||||
|
property: secret_access_key
|
||||||
|
---
|
||||||
|
apiVersion: barmancloud.cnpg.io/v1
|
||||||
|
kind: ObjectStore
|
||||||
|
metadata:
|
||||||
|
name: authentik-postgres
|
||||||
|
annotations:
|
||||||
|
# The CRD comes with the plugin's own app
|
||||||
|
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||||
|
spec:
|
||||||
|
retentionPolicy: 30d
|
||||||
|
configuration:
|
||||||
|
destinationPath: s3://cnpg-authentik/
|
||||||
|
endpointURL: http://192.168.1.99:3900
|
||||||
|
s3Credentials:
|
||||||
|
accessKeyId:
|
||||||
|
name: authentik-postgres-s3
|
||||||
|
key: ACCESS_KEY_ID
|
||||||
|
secretAccessKey:
|
||||||
|
name: authentik-postgres-s3
|
||||||
|
key: ACCESS_SECRET_KEY
|
||||||
|
region:
|
||||||
|
name: authentik-postgres-s3
|
||||||
|
key: REGION
|
||||||
|
wal:
|
||||||
|
compression: gzip
|
||||||
|
data:
|
||||||
|
compression: gzip
|
||||||
|
instanceSidecarConfiguration:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
env:
|
||||||
|
# Newer boto3 adds checksums that S3-compatible stores can reject;
|
||||||
|
# the plugin's docs suggest sending them only when required.
|
||||||
|
- name: AWS_REQUEST_CHECKSUM_CALCULATION
|
||||||
|
value: when_required
|
||||||
|
- name: AWS_RESPONSE_CHECKSUM_VALIDATION
|
||||||
|
value: when_required
|
||||||
|
---
|
||||||
|
apiVersion: postgresql.cnpg.io/v1
|
||||||
|
kind: ScheduledBackup
|
||||||
|
metadata:
|
||||||
|
name: authentik-postgres-daily
|
||||||
|
spec:
|
||||||
|
cluster:
|
||||||
|
name: authentik-postgres
|
||||||
|
# 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right
|
||||||
|
# away, so WAL archiving has a base backup to start from.
|
||||||
|
schedule: "0 0 3 * * *"
|
||||||
|
immediate: true
|
||||||
|
backupOwnerReference: self
|
||||||
|
method: plugin
|
||||||
|
pluginConfiguration:
|
||||||
|
name: barman-cloud.cloudnative-pg.io
|
||||||
@@ -6,12 +6,20 @@ spec:
|
|||||||
instances: 1
|
instances: 1
|
||||||
# Pinned so a new operator default (PostgreSQL 18 since CNPG 1.26.3) can't
|
# Pinned so a new operator default (PostgreSQL 18 since CNPG 1.26.3) can't
|
||||||
# start an in-place major upgrade. CNPG filled this in at creation.
|
# start an in-place major upgrade. CNPG filled this in at creation.
|
||||||
imageName: ghcr.io/cloudnative-pg/postgresql:17.4
|
imageName: ghcr.io/cloudnative-pg/postgresql:17.11
|
||||||
primaryUpdateStrategy: unsupervised
|
primaryUpdateStrategy: unsupervised
|
||||||
|
# WAL archiving and base backups to Garage, see postgresql-backup.yaml
|
||||||
|
plugins:
|
||||||
|
- name: barman-cloud.cloudnative-pg.io
|
||||||
|
isWALArchiver: true
|
||||||
|
parameters:
|
||||||
|
barmanObjectName: authentik-postgres
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 50m
|
cpu: 50m
|
||||||
memory: 256Mi
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
bootstrap:
|
bootstrap:
|
||||||
initdb:
|
initdb:
|
||||||
database: authentik
|
database: authentik
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
|
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
|
||||||
# changes:
|
# changes:
|
||||||
# - one backup (backups/<name>/) and the kopia repositories of the chosen
|
# - one backup (backups/<name>/) and the kopia repositories of the chosen
|
||||||
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with
|
# namespaces (kopia/<namespace>/) are copied read-only from the NAS with
|
||||||
# rclone into a Docker volume. A second Velero on the real bucket could
|
# rclone into a Docker volume. A second Velero on the real bucket could
|
||||||
# run kopia maintenance on the same repositories as the cluster's own
|
# run kopia maintenance on the same repositories as the cluster's own
|
||||||
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
|
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
|
||||||
@@ -19,7 +19,7 @@
|
|||||||
# Defaults: the newest Completed daily-backup, and ntfy freshrss
|
# Defaults: the newest Completed daily-backup, and ntfy freshrss
|
||||||
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
|
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
|
||||||
#
|
#
|
||||||
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
|
# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
|
||||||
# into a temporary directory, and the copied backup holds every Secret of
|
# into a temporary directory, and the copied backup holds every Secret of
|
||||||
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
|
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
|
||||||
# cluster (kubeconfig in the printed directory) until you run it with
|
# cluster (kubeconfig in the printed directory) until you run it with
|
||||||
@@ -33,8 +33,6 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
|
PROD_KUBECONFIG=${PROD_KUBECONFIG:-$HOME/.kube/homelab-config}
|
||||||
MINIO=${MINIO:-http://192.168.1.99:9000}
|
|
||||||
BUCKET=velero-backups
|
|
||||||
PORT=${PORT:-16443}
|
PORT=${PORT:-16443}
|
||||||
K3S=rancher/k3s:v1.34.9-k3s1
|
K3S=rancher/k3s:v1.34.9-k3s1
|
||||||
RCLONE=rclone/rclone:1.75.1
|
RCLONE=rclone/rclone:1.75.1
|
||||||
@@ -73,6 +71,18 @@ wait_for() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
|
||||||
|
# request signed for the wrong region (rclone defaults to us-east-1); MinIO
|
||||||
|
# did so with a bare 400 on HEAD.
|
||||||
|
if [[ -n ${FROM_VOLUME:-} ]]; then
|
||||||
|
BUCKET=${BUCKET:-velero-backups}
|
||||||
|
else
|
||||||
|
bsl=$(pk -n velero get backupstoragelocations default -o json)
|
||||||
|
S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
|
||||||
|
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
|
||||||
|
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
|
||||||
|
fi
|
||||||
|
|
||||||
backup=${1:-}
|
backup=${1:-}
|
||||||
shift || true
|
shift || true
|
||||||
namespaces=("$@")
|
namespaces=("$@")
|
||||||
@@ -91,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
|
|||||||
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
|
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
|
||||||
else
|
else
|
||||||
echo "== copy from the NAS (read-only)"
|
echo "== copy from the NAS (read-only)"
|
||||||
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
||||||
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
||||||
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
||||||
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\n' "$MINIO" >> "$WORK/nas.env")
|
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
|
||||||
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
||||||
copy "backups/$backup"
|
copy "backups/$backup"
|
||||||
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
||||||
|
|||||||
@@ -54,3 +54,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# cert-manager's three pods reach each other, DNS and the API server (the
|
||||||
|
# node, port 16443), and take connections only from each other and the
|
||||||
|
# node: the API server calls the webhook from there and the kubelet probes
|
||||||
|
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). Nothing scrapes the metrics on
|
||||||
|
# 9402. The HTTP-01 solver pods run in each Certificate's own namespace
|
||||||
|
# (acme-http01-solver in <namespace>/networkpolicy.yaml), not here.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
---
|
||||||
|
# The controller talks to Let's Encrypt (443), and before it lets Let's
|
||||||
|
# Encrypt check a challenge it fetches http://<host>/.well-known/... itself
|
||||||
|
# (80). Every certificate's host has a public address, node001's or the
|
||||||
|
# home router's (checked 2026-10-26), so the LAN stays out.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: controller-to-internet
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: cert-manager
|
||||||
|
app.kubernetes.io/component: controller
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
- port: 443
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# The CloudNativePG operator reaches DNS, the API server (the node, port
|
||||||
|
# 16443) and the PostgreSQL instances it runs in other namespaces: the
|
||||||
|
# instance manager's status port (8000) and PostgreSQL (5432), which the
|
||||||
|
# instances' own policies let in (postgres-cnpg in authentik and immich).
|
||||||
|
# Besides its own namespace it takes connections only from the node: the
|
||||||
|
# API server calls its webhook (9443) from there and the kubelet probes
|
||||||
|
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). The chart's Application
|
||||||
|
# (apps/templates/cloudnativepg-helm.yaml) only installs the chart, so
|
||||||
|
# this folder gets an Application of its own, cloudnative-pg-folder.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
podSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: cnpg.io/cluster
|
||||||
|
operator: Exists
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
- port: 5432
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# Nothing runs in default (2026-10-26: only the kubernetes Service, whose
|
||||||
|
# endpoint is the API server on the node, which this doesn't touch). A pod
|
||||||
|
# started here, by `kubectl run` or a manifest without a namespace, gets
|
||||||
|
# no network at all, instead of reaching the LAN, Vault and every app.
|
||||||
|
# Anything that needs a network gets a namespace and a policy of its own.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default-deny
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
+1
-1
@@ -13,7 +13,7 @@ spec:
|
|||||||
restartPolicy: Never
|
restartPolicy: Never
|
||||||
containers:
|
containers:
|
||||||
- name: sync
|
- name: sync
|
||||||
image: python:3.12-alpine
|
image: python:3.15-alpine
|
||||||
command:
|
command:
|
||||||
- sh
|
- sh
|
||||||
- -c
|
- -c
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ records:
|
|||||||
- name: "ldap"
|
- name: "ldap"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
content: "62.41.86.27"
|
content: "62.41.86.53"
|
||||||
- name: "argocd"
|
- name: "argocd"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
@@ -60,7 +60,7 @@ records:
|
|||||||
- name: "s2"
|
- name: "s2"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
content: "62.41.86.27"
|
content: "62.41.86.53"
|
||||||
# ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
|
# ntfy for Gatus alerts, one next to each Gatus. Same reasoning as s2.
|
||||||
- name: "ntfy1"
|
- name: "ntfy1"
|
||||||
expire: 300
|
expire: 300
|
||||||
@@ -69,7 +69,7 @@ records:
|
|||||||
- name: "ntfy2"
|
- name: "ntfy2"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: A
|
type: A
|
||||||
content: "62.41.86.27"
|
content: "62.41.86.53"
|
||||||
- name: "@"
|
- name: "@"
|
||||||
expire: 300
|
expire: 300
|
||||||
type: MX
|
type: MX
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ resources:
|
|||||||
- external-secret.yaml
|
- external-secret.yaml
|
||||||
- rbac.yaml
|
- rbac.yaml
|
||||||
- cronjob.yaml
|
- cronjob.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
|
|
||||||
configMapGenerator:
|
configMapGenerator:
|
||||||
- name: dns-sync-script
|
- name: dns-sync-script
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# The only pod here is the dns-sync Job (a PostSync hook): it installs its
|
||||||
|
# Python packages from PyPI and pushes domains/ to TransIP's API, so it
|
||||||
|
# reaches DNS and the internet (not the LAN, the API server or other
|
||||||
|
# namespaces). Nothing connects to it.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
ports:
|
||||||
|
- port: 443
|
||||||
@@ -27,7 +27,9 @@ view, and everything is shared with everyone automatically.
|
|||||||
- **immich has no user groups**; albums are shared per user, as viewer or
|
- **immich has no user groups**; albums are shared per user, as viewer or
|
||||||
editor. The owner and editors of an album may add users and change
|
editor. The owner and editors of an album may add users and change
|
||||||
their roles. Search only covers your own photos and those of partners
|
their roles. Search only covers your own photos and those of partners
|
||||||
you show in your timeline.
|
you show in your timeline. People (face recognition) are per user too,
|
||||||
|
and a partner's library doesn't bring its people along; since 3.3 a
|
||||||
|
user can share people with the users in their cluster group.
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
@@ -46,7 +48,10 @@ view, and everything is shared with everyone automatically.
|
|||||||
every user into every album of the group account (`fotoupload@`), as
|
every user into every album of the group account (`fotoupload@`), as
|
||||||
editor or viewer by quota; the group account's library as a partner of
|
editor or viewer by quota; the group account's library as a partner of
|
||||||
everyone, shown in the timeline; album invitations for albums older than
|
everyone, shown in the timeline; album invitations for albums older than
|
||||||
a week hidden; sessions of OAuth users older than 30 days ended. Albums
|
a week hidden; sessions of OAuth users older than 30 days ended;
|
||||||
|
everyone in the group account's cluster group with its people shared
|
||||||
|
(names editable by those who may upload, added 2026-10-28, when nobody
|
||||||
|
but the group account could search by face). Albums
|
||||||
that kaderleden create get the group account as editor first (in the
|
that kaderleden create get the group account as editor first (in the
|
||||||
database, since the API only lets an album's owner or editors share
|
database, since the API only lets an album's owner or editors share
|
||||||
it), so they go to everyone too; names starting with "Privé"/"Prive"
|
it), so they go to everyone too; names starting with "Privé"/"Prive"
|
||||||
@@ -123,15 +128,13 @@ D. auth.scoutingmbg.nl (2026-10-25)
|
|||||||
|
|
||||||
## Open points
|
## Open points
|
||||||
|
|
||||||
- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
|
- DNS (2026-10-24/25): the group's DNS host (ns1/ns2.webhosted.nl) pointed
|
||||||
foto.scoutingmbg.nl from a CNAME to `mbg.hensen.io` (the home IP) to a
|
scoutingmbg.nl, www, foto and auth at node001 (rubenhensen.nl); those are
|
||||||
CNAME to `rubenhensen.nl`, which is node001 on Hetzner. Only ns2 has it
|
all the records it moved, and node001 tunnels all four home. ns1 lagged
|
||||||
(serial 2026102471); ns1 is stuck on serial 2026060640 from June and
|
behind ns2 (a zone serial from June) until 2026-10-24 16:50 UTC. Nothing
|
||||||
still answers `mbg.hensen.io`. node001 now tunnels foto.scoutingmbg.nl to
|
uses mbg.hensen.io anymore, so hensen.io can expire. The SPF record's
|
||||||
the home cluster like its other hosts, so both answers work. Keep
|
`a` now means node001, which is fine: the WordPress site sends no mail
|
||||||
hensen.io until ns1 has caught up; ask the DNS host to fix ns1, and
|
and the group's mail goes through Google (`mx`)
|
||||||
decide whether foto should stay behind node001 or point straight home
|
|
||||||
(CNAME `scoutingmbg.nl`)
|
|
||||||
- Public shared links become public for real once sol-auth is gone.
|
- Public shared links become public for real once sol-auth is gone.
|
||||||
- Photos leiding back up from their phones stay private until they add
|
- Photos leiding back up from their phones stay private until they add
|
||||||
them to a shared album.
|
them to a shared album.
|
||||||
@@ -38,14 +38,15 @@ between them live.
|
|||||||
|
|
||||||
## Before: shrink what has to move
|
## Before: shrink what has to move
|
||||||
|
|
||||||
- [ ] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
- [x] Delete stale snapshots. 188 snapshots hold 334 GiB, 214 GiB of it in
|
||||||
snapshots from before October 2026, nearly all on immich-claim
|
snapshots from before October 2026, nearly all on immich-claim
|
||||||
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
|
(pvc-75d269dd): a 72.5 GiB one from 2024-12-20, 71.3 GiB from
|
||||||
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
|
2025-03-02, daily ones from 2025-03-13 to 03-20 (a recurring job that no
|
||||||
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
||||||
don't depend on them. Deleting a snapshot merges it into the next one,
|
don't depend on them. Deleting a snapshot merges it into the next one,
|
||||||
which writes a lot, so do this in a quiet hour, one volume at a time.
|
which writes a lot, so do this in a quiet hour, one volume at a time.
|
||||||
Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
|
Happened on 2026-10-24, from a Claude Code session at Ruben's request
|
||||||
|
(the 45 snapshots from before October 2026, small volumes first): immich-claim's
|
||||||
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
||||||
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
||||||
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
||||||
|
|||||||
+236
-93
@@ -1,21 +1,27 @@
|
|||||||
# Roadmap
|
# Roadmap
|
||||||
|
|
||||||
Where the cluster stands and what is left, as of 2026-10-25. Takes over
|
Where the cluster stands and what is left, as of 2026-10-28. Takes over
|
||||||
from wishlist.txt (January 2026, removed); its items are sorted in at the
|
from wishlist.txt (January 2026, removed); its items are sorted in at the
|
||||||
bottom.
|
bottom.
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
||||||
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
SATA SSD that is 54% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
||||||
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
39 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
||||||
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
|
volume has one replica. authentik 2026.8.5, Vault 2.2.1, immich 3.3.5 on
|
||||||
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
|
PostgreSQL 17, MariaDB 11.4.13. Memory requests and limits on everything
|
||||||
01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
|
but Longhorn, the Vault servers, calico and the LDAP outpost. Backups go
|
||||||
and s2 (home) watch each other and alert through ntfy; s2 also gets an
|
to the NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to
|
||||||
hourly report on the backups and Vault's seal from inside the cluster.
|
Garage (MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90
|
||||||
Every app namespace has NetworkPolicies, and CI renders and validates all
|
days), and CNPG to Garage (a base backup at 03:00, the WAL every 5
|
||||||
apps on every PR. node001 runs mail and tunnels the public hosts home.
|
minutes, 30 days).
|
||||||
|
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
||||||
|
through ntfy; s2 also gets an hourly report from inside the cluster on the
|
||||||
|
backups, Vault's seal, the Argo CD apps and the CNPG clusters.
|
||||||
|
Every namespace with pods but kube-system has NetworkPolicies, and CI
|
||||||
|
renders and validates all apps on every PR. node001 runs mail and tunnels
|
||||||
|
the public hosts home.
|
||||||
|
|
||||||
## Follow-ups from this week
|
## Follow-ups from this week
|
||||||
|
|
||||||
@@ -25,18 +31,28 @@ docs/longhorn-second-ssd.md:
|
|||||||
- [x] Read `~/iosample.log` on opti1: the worst stretch was not the
|
- [x] Read `~/iosample.log` on opti1: the worst stretch was not the
|
||||||
backups but 8.5 hours of Longhorn merging immich-claim's deleted
|
backups but 8.5 hours of Longhorn merging immich-claim's deleted
|
||||||
snapshots (2026-10-24 14:30-23:15), see docs/upgrade-k8s-1.33.md
|
snapshots (2026-10-24 14:30-23:15), see docs/upgrade-k8s-1.33.md
|
||||||
- [ ] From 2026-10-27, if immich runs without problems: delete
|
- [x] From 2026-10-27, if immich runs without problems: delete
|
||||||
`immich/postgresql-cluster.yaml` (the PG14 cluster still runs, 20 GiB),
|
`immich/postgresql-cluster.yaml` (the PG14 cluster, 20 GiB), the
|
||||||
the Longhorn snapshots and backups `immich-pg14-pre-vchord` and
|
Longhorn snapshots and backups `immich-pg14-pre-vchord` and
|
||||||
`immich-pg17-pre-v3`, and both dumps in `~` on opti1. PostgreSQL 14
|
`immich-pg17-pre-v3`, and both dumps in `~` on opti1. Done 2026-10-28
|
||||||
reaches end of life on 2026-11-12
|
(PR #61), with the PG14 volume's other Longhorn backups.
|
||||||
|
`immich-pg17-pre-v3` only ever was a snapshot
|
||||||
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
||||||
(`backend-*.tar`, `dqlite-pre-*`)
|
(`backend-*.tar`, `dqlite-pre-*`)
|
||||||
- [ ] Log in to immich as a jeugdlid (quota 0)
|
- [ ] Log in to immich as a jeugdlid (quota 0): no uploads, albums as
|
||||||
- [ ] Back to 1.36/stable once it is at v1.36.5 or later. The snap is held,
|
viewer, people searchable but not renamable
|
||||||
so microk8s gets no updates at all until someone refreshes it
|
- [x] Searching by face: people are per user in immich, and a partner's
|
||||||
|
library doesn't bring its people along, so since the personal accounts
|
||||||
|
only the group account could. share-sync now puts everyone in the
|
||||||
|
group account's cluster group and shares its people (immich 3.3's
|
||||||
|
person sharing; PR #59, 2026-10-28): 4430 people, 169 named, for each
|
||||||
|
of the 6 accounts, with write access for those who may upload
|
||||||
|
- [ ] Back to 1.36/stable once it is at v1.36.5 or later (still v1.36.2 on
|
||||||
|
2026-10-25). The snap is held, so microk8s gets no updates at all until
|
||||||
|
someone refreshes it
|
||||||
- [ ] Second SSD, docs/longhorn-second-ssd.md
|
- [ ] Second SSD, docs/longhorn-second-ssd.md
|
||||||
- [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
|
- [x] `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML (PR #39,
|
||||||
|
2026-10-25; the models are cached on a volume)
|
||||||
|
|
||||||
## Next, in order
|
## Next, in order
|
||||||
|
|
||||||
@@ -48,8 +64,9 @@ group's ns2: ns1 is stuck on an old serial and still answers
|
|||||||
mbg.hensen.io (docs/immich-scouts-online.md).
|
mbg.hensen.io (docs/immich-scouts-online.md).
|
||||||
|
|
||||||
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
||||||
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
|
- [x] ns1 caught up on 2026-10-24 16:50 UTC; both ns1 and ns2 answer
|
||||||
rubenhensen.nl too
|
`rubenhensen.nl` for foto (checked 2026-10-25), and nothing in the
|
||||||
|
cluster uses hensen.io anymore. hensen.io can expire
|
||||||
|
|
||||||
### 2. Off-site backups
|
### 2. Off-site backups
|
||||||
|
|
||||||
@@ -60,25 +77,28 @@ belong to the group.
|
|||||||
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
|
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
|
||||||
friend.
|
friend.
|
||||||
|
|
||||||
- [ ] Ours to his bucket: Hyper Backup on the Synology, destination "S3
|
- [ ] Ours to his bucket: `rclone sync` from Garage's `velero-backups`
|
||||||
Storage" with his endpoint as a custom server. Client-side encryption
|
(~190 GB) to his bucket through an rclone `crypt` remote, in Docker on
|
||||||
on: the kopia repositories use Velero's default password
|
the NAS like the copy from MinIO (see 5). Encrypted on our side: the
|
||||||
(`static-passw0rd`), so without it he could read everything, the
|
kopia repositories use Velero's default password (`static-passw0rd`),
|
||||||
group's photos included. Keep the encryption key next to the Vault
|
so without it he could read everything, the group's photos included.
|
||||||
unseal key. Source: the shared folder with MinIO's `velero-backups`
|
Keep the crypt passwords next to the Vault unseal key. Daily after the
|
||||||
bucket (~150-200 GB). Daily after the Velero backup, Smart Recycle,
|
Velero backup, ntfy on failure. Hyper Backup, the earlier plan, backs up
|
||||||
mail on failure
|
shared folders; Garage keeps its objects as blocks plus a metadata
|
||||||
|
database, and a copy of that folder taken while Garage runs is not
|
||||||
|
consistent
|
||||||
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
|
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
|
||||||
now, 500 GB to grow), and versioning or object lock on the bucket, so
|
now, 500 GB to grow), and versioning or object lock on the bucket, so
|
||||||
whoever takes over the NAS can't delete the off-site copy as well
|
whoever takes over the NAS can't delete the off-site copy as well
|
||||||
- [ ] His backups with us: Garage (not MinIO, see 5) in Container Manager
|
- [ ] His backups with us: a bucket and key for him, with a quota, in the
|
||||||
on the Synology, in its own shared folder, with a bucket and key for him
|
Garage on the NAS (`nas/garage`, see 5). Only the S3 API public, as
|
||||||
and a quota. Only the S3 API public, as `s3.rubenhensen.nl` through
|
`s3.rubenhensen.nl` through node001 and Envoy with a cert-manager
|
||||||
node001 and Envoy with a cert-manager certificate; the admin API stays
|
certificate; the admin API stays on the LAN. The cluster side (DNS,
|
||||||
on the LAN. The cluster side (DNS, node001 tunnel, Certificate,
|
node001 tunnel, Certificate, HTTPRoute) goes in git. Needed: how much
|
||||||
HTTPRoute) goes in git. Needed: how much space he wants, free space on
|
space he wants, free space on the NAS
|
||||||
the NAS
|
- [x] Which copy of the immich library goes off-site: Velero's, with the
|
||||||
- [ ] Decide which copy of the immich library goes off-site (see 5)
|
rest of the bucket (decided 2026-10-26). It is most of the ~190 GB, so
|
||||||
|
ask him for room for it
|
||||||
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
||||||
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
|
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
|
||||||
Planned on 2026-10-25, not run: a second Velero on the real bucket is
|
Planned on 2026-10-25, not run: a second Velero on the real bucket is
|
||||||
@@ -97,8 +117,20 @@ friend.
|
|||||||
ReadOnly location: data restored byte for byte (same sha256), deployment
|
ReadOnly location: data restored byte for byte (same sha256), deployment
|
||||||
available, everything removed afterwards. Two pitfalls found on the way
|
available, everything removed afterwards. Two pitfalls found on the way
|
||||||
are in the script's header. Not yet run against the real backups:
|
are in the script's header. Not yet run against the real backups:
|
||||||
- [ ] Run `backup-testing/restore-test.sh` (it reads the MinIO key from
|
- [x] Ran `backup-testing/restore-test.sh` on 2026-10-25 against
|
||||||
the cluster), then schedule it
|
daily-backup-20261025010027 (ntfy, freshrss, actual-budget): every pod
|
||||||
|
volume came back with its backed-up byte count (actual-budget 8.4 MB,
|
||||||
|
freshrss 293 MB, ntfy 180 KB) and all deployments became available. The
|
||||||
|
first run failed on a 400 from MinIO: rclone signed for us-east-1 while
|
||||||
|
MinIO runs region `minio`; the script now reads endpoint, region and
|
||||||
|
bucket from Velero's location. Copying freshrss's kopia repository took
|
||||||
|
14 of the ~17 minutes
|
||||||
|
- [x] Ran it again on 2026-10-26 against the copy in Garage
|
||||||
|
(daily-backup-20261026010028, same namespaces), before Velero switched
|
||||||
|
over (see 5): passed, in 5 minutes instead of 17
|
||||||
|
- [ ] Run it on a schedule. It needs Docker and an S3 key, so not in the
|
||||||
|
cluster itself: the Mac (launchd) or node001, ideally with a read-only
|
||||||
|
Garage key (`garage key create`, `garage bucket allow --read`)
|
||||||
|
|
||||||
### 3. authentik 2026.2.1 → 2026.8.5
|
### 3. authentik 2026.2.1 → 2026.8.5
|
||||||
|
|
||||||
@@ -153,8 +185,8 @@ unseals it.
|
|||||||
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
||||||
doesn't exist, so the OIDC mount's listing visibility was never set.
|
doesn't exist, so the OIDC mount's listing visibility was never set.
|
||||||
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
||||||
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
|
- [x] There is one unseal key (1 share, threshold 1), kept safe outside
|
||||||
places outside this cluster; without it the data in Vault is lost
|
this cluster (2026-10-25); without it the data in Vault is lost
|
||||||
- [x] Unsealing is in the SSD install steps
|
- [x] Unsealing is in the SSD install steps
|
||||||
- [ ] After a week without problems: delete the three snapshots
|
- [ ] After a week without problems: delete the three snapshots
|
||||||
|
|
||||||
@@ -181,18 +213,55 @@ Findings from 2026-10-25:
|
|||||||
- Velero copies the CNPG and MariaDB data directories file by file while
|
- Velero copies the CNPG and MariaDB data directories file by file while
|
||||||
the databases run, which is not a consistent copy. The Longhorn backups
|
the databases run, which is not a consistent copy. The Longhorn backups
|
||||||
start from a snapshot and are crash-consistent. CNPG can back up to
|
start from a snapshot and are crash-consistent. CNPG can back up to
|
||||||
MinIO itself (barman cloud plugin), with point-in-time recovery.
|
Garage itself (barman cloud plugin), with point-in-time recovery.
|
||||||
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
|
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
|
||||||
maintenance job pods) was fixed that afternoon by
|
maintenance job pods) was fixed that afternoon by
|
||||||
`velero/skip-emptydir-policy.yaml`.
|
`velero/skip-emptydir-policy.yaml`.
|
||||||
|
|
||||||
- [ ] Whether the immich library stays in Velero. It stays for now
|
- [x] Whether the immich library stays in Velero: it stays, since
|
||||||
(2026-10-25); decide again with the off-site copy (2)
|
Velero's bucket is what goes off-site (2; decided 2026-10-26)
|
||||||
- [ ] CNPG backups through the barman cloud plugin
|
- [x] CNPG backups through the barman cloud plugin (PR #56, merged
|
||||||
- [ ] Velero's target, MinIO on the NAS, is no longer maintained: no
|
2026-10-26 22:43 UTC): authentik's and immich's PostgreSQL 17 back up
|
||||||
|
to their own bucket in Garage (`cnpg-authentik`, `cnpg-immich`), a base
|
||||||
|
backup at 03:00 UTC and the WAL every 5 minutes, kept 30 days. Adding
|
||||||
|
the sidecar restarted both instances; authentik was down from 22:44 to
|
||||||
|
22:48, since the old pod waited for its connections to close. The first
|
||||||
|
base backups: 46 MB in 29 s and 776 MB in 3.5 minutes. immich's
|
||||||
|
`immediate` backup ran before the plugin was up and failed; a manual one
|
||||||
|
(`immich-postgres-17-first`) went through. The 03:00 base backups of
|
||||||
|
10-27 and 10-28 completed for both; the failed backup is deleted
|
||||||
|
- [ ] Restore one of them into a new Cluster (`bootstrap.recovery` from
|
||||||
|
the ObjectStore, to a point in time) to see that it works. After that,
|
||||||
|
decide whether Velero still needs to copy the CNPG data directories
|
||||||
|
- [ ] Take `cnpg-authentik` and `cnpg-immich` along in the off-site copy
|
||||||
|
(2)
|
||||||
|
- [x] Velero's target, MinIO on the NAS, is no longer maintained: no
|
||||||
community images since October 2025, maintenance mode since December
|
community images since October 2025, maintenance mode since December
|
||||||
2025, and the repository is archived. Move Velero to the Garage from 2
|
2025, and the repository is archived. Since 2026-10-26 Velero backs up
|
||||||
and switch MinIO off
|
to Garage on the same NAS (`nas/garage`, PR #35). The bucket was copied
|
||||||
|
with rclone (185 GiB in about 6 hours, then a catch-up sync just before
|
||||||
|
the switch), so the existing backups and kopia repositories carried on:
|
||||||
|
after the switch all 55 backups were still listed, and a manual backup
|
||||||
|
of ntfy found the existing repository and used the night's snapshot as
|
||||||
|
its parent. Two things made the NAS slow during the copy: Garage on
|
||||||
|
LMDB with `metadata_fsync`, ~270 disk writes/s while idle (now SQLite,
|
||||||
|
see `nas/garage/garage.toml`), and MinIO's scanner, which kept one disk
|
||||||
|
at 97% busy. The scanner is now at `slowest` (`mc admin config set local
|
||||||
|
scanner speed=slowest` in the MinIO container); it may have slowed
|
||||||
|
Velero's backups to MinIO as well
|
||||||
|
- [ ] Switch MinIO off once Velero has run on Garage for a week (from
|
||||||
|
2026-11-02): stop the container, then delete `/volume1/minio` (343 GiB)
|
||||||
|
and the hand-made Secret `velero/cloud-credentials` (MinIO's key). Only
|
||||||
|
MinIO still has the kopia repositories of namespaces that are gone
|
||||||
|
(~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
|
||||||
|
mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
|
||||||
|
whether any of them is worth keeping. The dailies of 10-27 and 10-28
|
||||||
|
on Garage completed without errors
|
||||||
|
- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
|
||||||
|
2026-10-26 next to a 270 MB database, and going by the folders in
|
||||||
|
`meta/snapshots`, each 6-hourly metadata snapshot takes about 70
|
||||||
|
minutes. Not growing; check again after a week of normal use. 1.8 GB
|
||||||
|
on 2026-10-28
|
||||||
|
|
||||||
### 6. Monitoring
|
### 6. Monitoring
|
||||||
|
|
||||||
@@ -261,12 +330,75 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
OIDC logins of Vault and FreshRSS reach authentik, all 24
|
OIDC logins of Vault and FreshRSS reach authentik, all 24
|
||||||
ExternalSecrets refresh, raft intact; Gitea no longer reaches the NAS
|
ExternalSecrets refresh, raft intact; Gitea no longer reaches the NAS
|
||||||
or Vault, and a pod in `default` no longer reaches Vault or Gitea
|
or Vault, and a pod in `default` no longer reaches Vault or Gitea
|
||||||
- [ ] The infrastructure namespaces (cert-manager, cnpg-system,
|
- [x] Certificates couldn't renew in those 11 namespaces: Envoy may only
|
||||||
|
reach each app's own port, and cert-manager's HTTP-01 solver listens on
|
||||||
|
8089. ynab's renewal hung from 2026-10-26 17:35 UTC; a policy for the
|
||||||
|
solver pods, applied by hand in actual-budget that evening, let it
|
||||||
|
through within 20 seconds (now valid until 2027-01-24). PR #51 adds it
|
||||||
|
to all 11 (merged 2026-10-26 22:27 UTC). Without it the next renewals
|
||||||
|
would hang too; the first real test is gitea's on 11-15, then
|
||||||
|
bolletjestrui on 11-18 and the rest from 12-03
|
||||||
|
- [x] The infrastructure namespaces (cert-manager, cnpg-system,
|
||||||
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
||||||
default) still have none; `default` can reach the LAN and everything
|
default), merged 2026-10-26: PR #54 for envoy-gateway-system (22:31
|
||||||
without a policy
|
UTC), PR #53 for the rest (22:42 UTC; one commit per namespace, and
|
||||||
- [ ] Requests and memory limits, the big ones first (immich, authentik,
|
`default` denies everything). Checked afterwards: all 16 public hosts
|
||||||
Argo CD's repo-server)
|
answer as before; server-side dry runs reach the CNPG, ESO,
|
||||||
|
cert-manager and MariaDB webhooks (all fail closed); Velero's location
|
||||||
|
Available, every ExternalSecret synced, the dns-sync job ran, no errors
|
||||||
|
from the operators after the barman plugin had started. Velero's policy
|
||||||
|
only lets it reach Garage, not MinIO's port 9000
|
||||||
|
- [x] Requests: PR #33, merged 2026-10-25 11:23 UTC, for everything that
|
||||||
|
had none except the Vault server, from `kubectl top` every 5 minutes
|
||||||
|
from 03:13 to 08:40 UTC (highest memory plus 10%, CPU at p95, 10m
|
||||||
|
floor). Rolled out in about eight minutes; both CNPG instances
|
||||||
|
restarted and came back healthy, the OIDC endpoints, the logins and
|
||||||
|
immich answered, no OOM kills. The node now has 3770m CPU (62%, of which
|
||||||
|
1440m is Longhorn's two instance managers) and 9158Mi memory (28%)
|
||||||
|
requested. Still without requests: Longhorn (22 containers, its chart),
|
||||||
|
the Vault servers, the LDAP outpost, immich's PG14 cluster and three in
|
||||||
|
kube-system
|
||||||
|
- [x] Memory limits: PR #58, for everything that got requests in #33.
|
||||||
|
About twice the highest memory from `kubectl top` every 5 minutes from
|
||||||
|
2026-10-26 22:29 to 2026-10-27 21:55 UTC or the night of #33, rounded
|
||||||
|
up; no CPU limits. No OOM kills that day. immich's machine learning
|
||||||
|
sat idle all day (240Mi), so on 2026-10-28 a test image went through
|
||||||
|
CLIP, face recognition and OCR inside the container: 884Mi with all
|
||||||
|
three loaded, limit 2Gi
|
||||||
|
- [ ] authentik's server fails its liveness probe while its database is
|
||||||
|
away (`/-/health/live/` answers 500 when PostgreSQL is down), so each
|
||||||
|
database restart gets the server killed too, as on 2026-10-27 for
|
||||||
|
PostgreSQL 17.11. The login only comes back once the server has
|
||||||
|
started again. A liveness probe that doesn't touch the database would
|
||||||
|
leave it at not ready
|
||||||
|
- [x] WordPress (scoutingmbg.nl) was OOMKilled at 08:27:59 UTC on
|
||||||
|
2026-10-25, its third restart: a crawler asked for 88 pages and assets
|
||||||
|
within a few seconds (normally ~6 a minute), Apache started a PHP worker
|
||||||
|
for each (~45-60 MB each, `MaxRequestWorkers` was the default 150), and
|
||||||
|
the container went past its 1Gi limit. Unrelated to the
|
||||||
|
NetworkPolicies. Now capped at 12 workers through the chart's
|
||||||
|
`apachePortsConfig` (ports.conf comes after mods-enabled/*.conf), checked
|
||||||
|
with `apache2ctl -t` in the same image first. Replaying the burst (90
|
||||||
|
page requests, 40 at a time): all 200, at most 13 Apache processes and
|
||||||
|
572Mi, no restart; the price is waiting (4.3s median, up to ~22s at the
|
||||||
|
peak) instead of a crash
|
||||||
|
- [ ] Optional, no plugins wanted on WordPress: a small nginx cache in the
|
||||||
|
cluster between Envoy and WordPress that keeps anonymous GETs for a few
|
||||||
|
minutes and passes anything with a WordPress cookie through. A page
|
||||||
|
renders in 0.4-0.7s when the site is quiet and OPcache is on, so the
|
||||||
|
waiting in a burst is queueing; the cap alone already prevents the crash
|
||||||
|
- [ ] Fewer plugins on the site (17 now; up to whoever runs it):
|
||||||
|
`litespeed-cache` only caches pages on a LiteSpeed server, so it does
|
||||||
|
little here, `wordpress-importer` is usually for a one-off import, and
|
||||||
|
`code-snippets` runs PHP from the dashboard
|
||||||
|
- [ ] Rate limiting per visitor in Envoy, on hold. Since PR #50
|
||||||
|
(2026-10-26) node001 passes the visitor's address on with the PROXY
|
||||||
|
protocol, and the apps see it: authentik logs it, Apache logs it for
|
||||||
|
WordPress, so Wordfence's lockout after 20 failed logins now hits one
|
||||||
|
visitor instead of everyone. Envoy Gateway's local rate limit can't
|
||||||
|
count per address (no `Distinct`); that takes its global rate limit,
|
||||||
|
with a Redis and a ratelimit service next to it. Not worth two more
|
||||||
|
components on the one node while the apps' own limits work
|
||||||
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
|
- [x] Probes for gitea (`/api/healthz`), freshrss (TCP, since
|
||||||
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
|
mod_auth_openidc answers 401 to non-browsers) and serpbear (`/login`)
|
||||||
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
|
(2026-10-25). Liveness only after 5 failures 30s apart, to ride out the
|
||||||
@@ -277,8 +409,10 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
|
|
||||||
### 8. Renovate and CI
|
### 8. Renovate and CI
|
||||||
|
|
||||||
10 Renovate PRs are open, the oldest from February, and nothing checks a
|
On 2026-10-25 10 Renovate PRs were open, the oldest from February, and
|
||||||
PR before Argo CD applies it.
|
nothing checked a PR before Argo CD applied it. All of them have since been
|
||||||
|
merged or closed (below), none are open (checked 2026-10-25), and CI checks
|
||||||
|
every PR.
|
||||||
|
|
||||||
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
|
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
|
||||||
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
|
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
|
||||||
@@ -302,7 +436,15 @@ PR before Argo CD applies it.
|
|||||||
fail the run; tested with a planted Secret and an untagged image), plus
|
fail the run; tested with a planted Secret and an untagged image), plus
|
||||||
PR annotations for containers without requests or probes. freshrss's
|
PR annotations for containers without requests or probes. freshrss's
|
||||||
`alpine/git` init container had no tag; now `v2.52.0`
|
`alpine/git` init container had no tag; now `v2.52.0`
|
||||||
- [ ] Automerge patch versions of low-risk apps now that CI runs
|
- [x] Automerge patch versions of low-risk apps now that CI runs: PR #52,
|
||||||
|
merged 2026-10-26 (ntfy, gatus, freshrss, serpbear, actual-budget and
|
||||||
|
CI's own tools; three days after the release, only on weekday evenings
|
||||||
|
and at the weekend, when the cluster can be reached). Nothing has
|
||||||
|
merged itself yet by 2026-10-28
|
||||||
|
- [x] MariaDB stays on 11.4 (PR #60, 2026-10-28): Renovate offered 11.8.9
|
||||||
|
as a minor update (#42, closed), but 11.8 can't be rolled back and has
|
||||||
|
community support until June 2028, against May 2029 for 11.4. Renovate
|
||||||
|
now only offers 11.4 patches; the next LTS step, 12.3, is by hand
|
||||||
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
|
- [x] Renovate listed external-snapshotter twice (#15 and #24): its
|
||||||
`argocd` manager and a custom `github-tags` regex both matched. The
|
`argocd` manager and a custom `github-tags` regex both matched. The
|
||||||
regex is gone; Renovate should close #15 by itself
|
regex is gone; Renovate should close #15 by itself
|
||||||
@@ -314,34 +456,30 @@ PR before Argo CD applies it.
|
|||||||
cover every app folder, a regex manager reads `imageName` in
|
cover every app folder, a regex manager reads `imageName` in
|
||||||
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
|
`*/postgresql-cluster*.yaml`, and major PostgreSQL updates are off
|
||||||
(those need pg_upgrade or a dump). Expect a batch of new PRs
|
(those need pg_upgrade or a dump). Expect a batch of new PRs
|
||||||
(2026-10-25; `renovate-config-validator` passes)
|
(2026-10-25). They only come once fewer than 10 Renovate PRs are open
|
||||||
- [ ] The open PRs, all validated on 2026-10-25 against main as it is now
|
(`config:recommended`'s `prConcurrentLimit`); until then they wait
|
||||||
(rebases requested, so the checks show on the PRs once Renovate has
|
under "Rate-Limited" on the Dependency Dashboard (#2)
|
||||||
pushed). Merging is yours; merging deploys:
|
- The other folders were left out with a `(?!…)` lookahead. Renovate
|
||||||
- #28 cert-manager v1.21.3: security release, "All users should
|
compiles these patterns with RE2, which has no lookahead, so it
|
||||||
upgrade" (drops the compromised eMudhra root, fixes an ARI renewal
|
rejected the config and stopped opening PRs (#32). Each manager now
|
||||||
loop and a private key copy path). Merge first
|
takes every folder and skips the others through its own
|
||||||
- #24 external-snapshotter v8.2.1 → v8.7.0: 8.6 moved VolumeGroupSnapshot
|
`ignorePaths`. Renovate 44.181.6 with RE2 loaded fails the old config
|
||||||
to v1 and 8.7 turns its feature gate on; the cluster already serves the
|
with #32's error and passes the new one, and its matcher picks the
|
||||||
v1 CRDs. Merge; close #15
|
same 166 files as the old pattern did
|
||||||
- #25 WordPress chart 0.15.0 → 0.16.7: the image is pinned
|
- [x] Merged 2026-10-25 and checked after the sync: #28 cert-manager
|
||||||
(7.1.3-php8.3-apache), so the render only changes labels and
|
v1.21.3 (security release; all three pods on v1.21.3), #24
|
||||||
checksums (a pod restart). Merge; #19 (0.15.1) is superseded
|
external-snapshotter v8.7.0 (its manifests ship the v8.6.0 image, as
|
||||||
- #26 Gitea 1.27.3 → 28.2.0 (Gitea renumbered after 1.27): security
|
v8.2.1 shipped v8.2.0; both replicas up, no errors), #25 WordPress chart
|
||||||
fixes; the breaking changes are to egress host lists
|
0.16.7 (image unchanged, site 200), #31 python 3.15.1 (the cluster check
|
||||||
(`ALLOWED_DOMAINS` and friends), none of which are set here. Database
|
ran clean on it). Closed: #19, #29, and #15 by Renovate. Later the same
|
||||||
migrations can't be undone, so merge after a fresh backup
|
day: #23 ntfy v2.34.0, #27 go-dqlite v3.0.4, #33 (see 7) and #26 Gitea
|
||||||
- #23 ntfy v2.28.0 → v2.34.0: six releases in 17 days; 2.31.0 had a
|
28.2.0 (migrations up to 357 ran clean, `/api/healthz` 200; no backup of
|
||||||
database load bug that ntfy.sh rolled back (fixed in 2.31.1). No
|
its own beforehand, the nightly ones are the way back)
|
||||||
breaking migrations. Fine, or wait a few days for 2.34 to settle
|
- [x] #17 external-secrets 2.14.0, merged 2026-10-25 11:50 UTC after a
|
||||||
- #17 external-secrets 2.13.0 → 2.14.0: no breaking changes in the
|
manual rebase (#33 had added values on the next line, which git counts
|
||||||
notes; every secret goes through it, so merge when there's time to
|
as a conflict). All three pods on v2.14.0, the store Ready, and 24 of 25
|
||||||
watch a force-sync
|
ExternalSecrets refreshed on a force-sync; the 25th is the generated
|
||||||
- #27 go-dqlite v3.0.4 for `tools/dqlite-vacuum`: builds; harmless
|
`gatus/backup-check-token`, which never refreshes by design
|
||||||
- #29 yaml.v3 for the same tool: builds, but the code imports yaml.v2,
|
|
||||||
so it only adds an unused requirement. Close
|
|
||||||
- #31 python 3.15.1 for `gatus/backup-check.yaml`: the check's tests
|
|
||||||
(made-up data for every failure) pass on that image. Merge
|
|
||||||
|
|
||||||
### 9. One node
|
### 9. One node
|
||||||
|
|
||||||
@@ -357,11 +495,16 @@ step after that, if downtime starts to matter.
|
|||||||
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
||||||
completed kopia maintenance jobs of repositories that no longer exist
|
completed kopia maintenance jobs of repositories that no longer exist
|
||||||
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
||||||
- [ ] `kubectl delete namespace secrets`, empty since its app went
|
- [ ] freshrss mounts `freshrss-claim2` and `freshrss-claim3` as whole
|
||||||
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
|
volumes over the file paths `data/config.custom.php` and
|
||||||
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
|
`data/config-user.custom.php`, so FreshRSS sees two empty directories
|
||||||
left alone they expire by 2027-01-16.
|
there and the volumes hold nothing (0 bytes in every backup). Drop both
|
||||||
`kubectl -n velero exec deploy/velero -c velero -- /velero backup delete $(kubectl -n velero get backups.velero.io -o json | jq -r '.items[] | select(.status.phase=="Failed") | .metadata.name' | xargs) --confirm`
|
mounts and the two PVCs, or make them ConfigMap files if custom config
|
||||||
|
is wanted
|
||||||
|
- [x] The namespace `secrets` was already gone on 2026-10-26
|
||||||
|
- [x] The Failed Velero backups (the dailies from 2026-09-26 to 10-21 and
|
||||||
|
the weeklies since 08-02; 40 by then, one had expired), deleted through
|
||||||
|
Velero on 2026-10-26. Left: 13 Completed, 3 PartiallyFailed
|
||||||
- [ ] The mariadb-operator Backup `mbgwp-pre-migration-backup`, a dump
|
- [ ] The mariadb-operator Backup `mbgwp-pre-migration-backup`, a dump
|
||||||
from 2026-02-22 on its own 10 GiB volume (which has a Longhorn backup
|
from 2026-02-22 on its own 10 GiB volume (which has a Longhorn backup
|
||||||
from 2026-10-22). The PVC has no owner, so it goes separately:
|
from 2026-10-22). The PVC has no owner, so it goes separately:
|
||||||
@@ -393,18 +536,18 @@ critical / high:
|
|||||||
|
|
||||||
| Image | Crit | High | What to do |
|
| Image | Crit | High | What to do |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; goes with the cleanup from 2026-10-27 |
|
| tensorchord/cloudnative-pgvecto.rs:14-v0.2.1 | 38 | 342 | immich's old PG14 cluster; deleted 2026-10-28 |
|
||||||
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. Renovate now offers 17.7; a `17.x-standard-trixie` image is more current |
|
| cloudnative-pg/postgresql:17.4 | 27 | 260 | authentik's database, a Debian 11 image. 17.11 since PR #40 (2026-10-27); not scanned again, a `17.x-standard-trixie` image is more current |
|
||||||
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
||||||
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
||||||
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
||||||
| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
|
| gitea/gitea:1.27.3 | 8 | 43 | now 28.2.0 (PR #26, 2026-10-25); not scanned again |
|
||||||
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
|
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
|
||||||
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
|
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
|
||||||
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
|
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
|
||||||
| coredns/coredns:1.10.1 | 5 | 70 | see below |
|
| coredns/coredns:1.10.1 | 5 | 70 | see below |
|
||||||
| dexidp/dex:v2.45.1, argocd v3.5.6, redis 8.2.3 | 4, 1, 2 | | come with Argo CD |
|
| dexidp/dex:v2.45.1, argocd v3.5.6, redis 8.2.3 | 4, 1, 2 | | come with Argo CD |
|
||||||
| mariadb:11.4.5 | 4 | 65 | the newest 11.4 patch; Renovate now covers it |
|
| mariadb:11.4.5 | 4 | 65 | 11.4.13 since PR #60; Renovate stays on 11.4 (it offered 11.8, PR #42) |
|
||||||
| calico v3.29.3 | 2-3 | ~50 | comes with microk8s |
|
| calico v3.29.3 | 2-3 | ~50 | comes with microk8s |
|
||||||
| goauthentik ldap / server 2026.8.5 | 3, 1 | 74, 65 | the next authentik patch |
|
| goauthentik ldap / server 2026.8.5 | 3, 1 | 74, 65 | the next authentik patch |
|
||||||
|
|
||||||
|
|||||||
@@ -169,8 +169,9 @@ pgvecto.rs. Done 2026-10-24, about an hour of immich downtime:
|
|||||||
v2.4.1 refused to start (it only accepts VectorChord <0.6); v2.5.0 and up
|
v2.4.1 refused to start (it only accepts VectorChord <0.6); v2.5.0 and up
|
||||||
accept <2, so immich went to v2.7.5, the last 2.x. It rebuilt both
|
accept <2, so immich went to v2.7.5, the last 2.x. It rebuilt both
|
||||||
indexes as `vchordrq` in about 1.5 minutes
|
indexes as `vchordrq` in about 1.5 minutes
|
||||||
- [ ] After a few days without problems: delete the old cluster
|
- [x] After a few days without problems: delete the old cluster
|
||||||
(`immich/postgresql-cluster.yaml`), the snapshot/backup and the dump.
|
(`immich/postgresql-cluster.yaml`), the snapshot/backup and the dump.
|
||||||
|
Done 2026-10-28 (PR #61), with the PG14 volume's Longhorn backups.
|
||||||
Rolling back means reverting the old volume to the snapshot, since its
|
Rolling back means reverting the old volume to the snapshot, since its
|
||||||
embeddings are now `real[]` without pgvecto.rs
|
embeddings are now `real[]` without pgvecto.rs
|
||||||
- [x] immich 3.x: v2.7.5 → v3.3.5 with chart 0.10.3 → 0.13.4 on
|
- [x] immich 3.x: v2.7.5 → v3.3.5 with chart 0.10.3 → 0.13.4 on
|
||||||
@@ -189,8 +190,8 @@ pgvecto.rs. Done 2026-10-24, about an hour of immich downtime:
|
|||||||
(v3.3.0), for faster ML with less memory; numerically the same models,
|
(v3.3.0), for faster ML with less memory; numerically the same models,
|
||||||
so no jobs to rerun. The model cache is an emptyDir, so the one-off
|
so no jobs to rerun. The model cache is an emptyDir, so the one-off
|
||||||
preparation repeats on every restart
|
preparation repeats on every restart
|
||||||
- [ ] After a few days without problems: delete the `immich-pg17-pre-v3`
|
- [x] After a few days without problems: delete the `immich-pg17-pre-v3`
|
||||||
snapshot and dump together with the PG14 leftovers above
|
snapshot and dump together with the PG14 leftovers above (2026-10-28)
|
||||||
|
|
||||||
## k8s-dqlite write stalls (found 2026-10-22)
|
## k8s-dqlite write stalls (found 2026-10-22)
|
||||||
|
|
||||||
@@ -282,6 +283,13 @@ whatever time of day the hourly schedule had drifted to.
|
|||||||
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
||||||
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
||||||
their leases at 01:31
|
their leases at 01:31
|
||||||
|
Per minute against dqlite's own errors (2026-10-24 12:35 to 2026-10-25
|
||||||
|
11:30, leaving out the CNPG range cancels): all 67 minutes with errors
|
||||||
|
had sda ~100% busy, single 2s windows at 200 ms to 2.2 s per write and
|
||||||
|
"full" pressure at 45-78%, against a median minute peak of 24 ms. The
|
||||||
|
worst minute after the merges was the rollout of eight merged PRs at
|
||||||
|
2026-10-25 11:25-11:27 (77 errors in one minute). The snapshot-controller,
|
||||||
|
CNPG and Envoy Gateway held their 60s leases through all of it
|
||||||
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
||||||
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
||||||
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# External Secrets' three pods reach each other, DNS and the API server
|
||||||
|
# (the node, port 16443); the controller also reads Vault
|
||||||
|
# (vault-active.vault.svc:8200, which vault/networkpolicy.yaml lets it
|
||||||
|
# into). Besides their own namespace they take connections only from the
|
||||||
|
# node: the API server calls the webhook from there and the kubelet probes
|
||||||
|
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). The Password generator behind
|
||||||
|
# gatus/backup-check-token needs no network. The chart's Application
|
||||||
|
# (apps/templates/external-secrets-helm.yaml) only installs the chart, so
|
||||||
|
# this folder gets an Application of its own, external-secrets-folder.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: controller-to-vault
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: external-secrets
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: vault
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: vault
|
||||||
|
ports:
|
||||||
|
- port: 8200
|
||||||
@@ -23,7 +23,7 @@ spec:
|
|||||||
io.kompose.service: freshrss
|
io.kompose.service: freshrss
|
||||||
spec:
|
spec:
|
||||||
initContainers:
|
initContainers:
|
||||||
- image: alpine/git:v2.52.0
|
- image: alpine/git:v2.54.0
|
||||||
name: git
|
name: git
|
||||||
command:
|
command:
|
||||||
- git
|
- git
|
||||||
@@ -74,7 +74,7 @@ spec:
|
|||||||
value: Europe/Amsterdam
|
value: Europe/Amsterdam
|
||||||
# 1.30.3 ships PHP 8.4.27, which fixes CVE-2026-109618 and
|
# 1.30.3 ships PHP 8.4.27, which fixes CVE-2026-109618 and
|
||||||
# CVE-2026-109619.
|
# CVE-2026-109619.
|
||||||
image: freshrss/freshrss:1.30.3
|
image: freshrss/freshrss:1.30.4
|
||||||
name: freshrss
|
name: freshrss
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 80
|
- containerPort: 80
|
||||||
@@ -83,6 +83,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 50m
|
cpu: 50m
|
||||||
memory: 176Mi
|
memory: 176Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
|
# TCP, because mod_auth_openidc answers 401 to anything that isn't a
|
||||||
# browser.
|
# browser.
|
||||||
readinessProbe:
|
readinessProbe:
|
||||||
|
|||||||
@@ -56,3 +56,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
# Envoy Gateway: the controller and the proxy it runs (envoyproxy.yaml).
|
||||||
|
# Both reach each other (the proxy gets its config from the controller on
|
||||||
|
# 18000) and DNS, and take connections from each other and the node: the
|
||||||
|
# API server calls the controller's topology webhook (9443) from there and
|
||||||
|
# the kubelet probes come from there too (10.1.7.64 is the node's Calico
|
||||||
|
# VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr). Nothing scrapes
|
||||||
|
# the metrics on 19001.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
namespace: envoy-gateway-system
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
---
|
||||||
|
# The proxy holds the node's ports 80 and 443 (10080 and 10443 inside) and
|
||||||
|
# takes connections there from anywhere: the internet through node001 and
|
||||||
|
# the router, the LAN, and pods that call a public hostname. It reaches
|
||||||
|
# every app it routes to (pods in any namespace; their own policies decide
|
||||||
|
# which port), Home Assistant on 192.168.1.202:8123 (outside the cluster,
|
||||||
|
# home-assistant/), and the internet on 443 for Longhorn's OIDC login
|
||||||
|
# (authentik's token endpoint, at its public address).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: proxy
|
||||||
|
namespace: envoy-gateway-system
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
app.kubernetes.io/component: proxy
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- ports:
|
||||||
|
- port: 10080
|
||||||
|
- port: 10443
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.202/32
|
||||||
|
ports:
|
||||||
|
- port: 8123
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
ports:
|
||||||
|
- port: 443
|
||||||
|
---
|
||||||
|
# Everything but the proxy (the controller, and the chart's certgen job on
|
||||||
|
# upgrades) reaches the API server (the node, port 16443). The controller
|
||||||
|
# also fetches the OIDC discovery document of Longhorn's SecurityPolicy
|
||||||
|
# from authentik's public address.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: control-plane
|
||||||
|
namespace: envoy-gateway-system
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: app.kubernetes.io/name
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- envoy
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
ports:
|
||||||
|
- port: 443
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# node001 passes TLS for the tunneled hosts through without opening it, so
|
||||||
|
# without this every request arrives from 46.224.26.65. Wordfence locks that
|
||||||
|
# address out after 20 failed logins (2026-10-26), and with it everyone who
|
||||||
|
# logs in to scoutingmbg.nl. node001 puts a PROXY header with the visitor's
|
||||||
|
# address in front of each connection; Envoy takes the address from there
|
||||||
|
# and adds it to X-Forwarded-For.
|
||||||
|
#
|
||||||
|
# Optional, because s2 and ntfy2 point straight at home and their clients
|
||||||
|
# send no header, and neither does node001's proxy on port 80. The price:
|
||||||
|
# whoever connects to the home address directly can send a header of their
|
||||||
|
# own and claim any address.
|
||||||
|
apiVersion: gateway.envoyproxy.io/v1alpha1
|
||||||
|
kind: ClientTrafficPolicy
|
||||||
|
metadata:
|
||||||
|
name: proxy-protocol
|
||||||
|
namespace: envoy-gateway-system
|
||||||
|
spec:
|
||||||
|
targetRefs:
|
||||||
|
- group: gateway.networking.k8s.io
|
||||||
|
kind: Gateway
|
||||||
|
name: homelab
|
||||||
|
proxyProtocol:
|
||||||
|
optional: true
|
||||||
+2
-2
@@ -62,7 +62,7 @@ node, both PowerDNS Authoritative. PowerDNS LUA records health-check
|
|||||||
addresses and only return the healthy ones:
|
addresses and only return the healthy ones:
|
||||||
|
|
||||||
```
|
```
|
||||||
@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.27'}})"
|
@ 30 IN LUA A "ifurlup('https://status.rubenhensen.nl/health', {{'46.224.26.65'}, {'62.41.86.53'}})"
|
||||||
```
|
```
|
||||||
|
|
||||||
The double braces make two groups. PowerDNS returns the first group that has
|
The double braces make two groups. PowerDNS returns the first group that has
|
||||||
@@ -85,7 +85,7 @@ Steps:
|
|||||||
|
|
||||||
1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add
|
1. Parent zone (`dns/domains/rubenhensen.nl.yaml`, synced to TransIP): add
|
||||||
`status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`,
|
`status NS ns-hetzner.rubenhensen.nl.` and `status NS ns-home.rubenhensen.nl.`,
|
||||||
plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.27).
|
plus A records for `ns-hetzner` (46.224.26.65) and `ns-home` (62.41.86.53).
|
||||||
First check that TransIP accepts NS records for a subdomain.
|
First check that TransIP accepts NS records for a subdomain.
|
||||||
2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and
|
2. Hetzner: run PowerDNS from node001.nix with `enable-lua-records=yes` and
|
||||||
the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp.
|
the bind backend. Use the same zone file as home. Open 53/udp and 53/tcp.
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: check
|
- name: check
|
||||||
image: python:3.13.16-alpine3.24
|
image: python:3.15.2-alpine3.24
|
||||||
command: ["python", "/script/check.py"]
|
command: ["python", "/script/check.py"]
|
||||||
env:
|
env:
|
||||||
- name: PYTHONDONTWRITEBYTECODE
|
- name: PYTHONDONTWRITEBYTECODE
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: gatus
|
- name: gatus
|
||||||
image: twinproduction/gatus:v5.37.0
|
image: twinproduction/gatus:v5.38.0
|
||||||
env:
|
env:
|
||||||
- name: GATUS_CONFIG_PATH
|
- name: GATUS_CONFIG_PATH
|
||||||
value: /config/config.yaml
|
value: /config/config.yaml
|
||||||
|
|||||||
@@ -83,3 +83,29 @@ spec:
|
|||||||
cidr: 192.168.1.204/32
|
cidr: 192.168.1.204/32
|
||||||
ports:
|
ports:
|
||||||
- port: 16443
|
- port: 16443
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -20,7 +20,7 @@ spec:
|
|||||||
fsGroup: 1000
|
fsGroup: 1000
|
||||||
containers:
|
containers:
|
||||||
- name: gitea
|
- name: gitea
|
||||||
image: gitea/gitea:1.27.3
|
image: gitea/gitea:28.3.0
|
||||||
env:
|
env:
|
||||||
- name: USER_UID
|
- name: USER_UID
|
||||||
value: "1000"
|
value: "1000"
|
||||||
@@ -54,6 +54,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 20m
|
cpu: 20m
|
||||||
memory: 336Mi
|
memory: 336Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
# /api/healthz checks the database and the cache.
|
# /api/healthz checks the database and the cache.
|
||||||
readinessProbe:
|
readinessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
|
|||||||
@@ -55,3 +55,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
+22
-5
@@ -67,6 +67,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 64Mi
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
image:
|
image:
|
||||||
repository: docker.io/valkey/valkey
|
repository: docker.io/valkey/valkey
|
||||||
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
tag: 9.1.2-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
||||||
@@ -93,6 +95,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: 100m
|
||||||
memory: 1280Mi
|
memory: 1280Mi
|
||||||
|
limits:
|
||||||
|
memory: 3Gi
|
||||||
# Node trusts its built-in roots, which still include the
|
# Node trusts its built-in roots, which still include the
|
||||||
# ones above the compromised eMudhra intermediates. Use
|
# ones above the compromised eMudhra intermediates. Use
|
||||||
# patched-ca-bundle through OpenSSL until Node ships a fix.
|
# patched-ca-bundle through OpenSSL until Node ships a fix.
|
||||||
@@ -117,16 +121,21 @@ spec:
|
|||||||
image:
|
image:
|
||||||
repository: ghcr.io/immich-app/immich-machine-learning
|
repository: ghcr.io/immich-app/immich-machine-learning
|
||||||
pullPolicy: IfNotPresent
|
pullPolicy: IfNotPresent
|
||||||
# Idle at night (234Mi); the models take over a GiB when
|
# Idle at 240Mi; 884Mi with CLIP, face recognition and
|
||||||
# it works.
|
# OCR loaded (2026-10-28, a test image through each).
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 50m
|
cpu: 50m
|
||||||
memory: 1Gi
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
env:
|
env:
|
||||||
TRANSFORMERS_CACHE: /cache
|
TRANSFORMERS_CACHE: /cache
|
||||||
HF_XET_CACHE: /cache/huggingface-xet
|
HF_XET_CACHE: /cache/huggingface-xet
|
||||||
MPLCONFIGDIR: /cache/matplotlib-config
|
MPLCONFIGDIR: /cache/matplotlib-config
|
||||||
|
# The optimized models of v3.3.0: faster, less memory,
|
||||||
|
# and the same results, so no jobs to run again.
|
||||||
|
MACHINE_LEARNING_MODEL_REVISION: v2
|
||||||
# Trust store without the compromised eMudhra
|
# Trust store without the compromised eMudhra
|
||||||
# intermediates; requests would otherwise use certifi.
|
# intermediates; requests would otherwise use certifi.
|
||||||
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
||||||
@@ -139,11 +148,19 @@ spec:
|
|||||||
globalMounts:
|
globalMounts:
|
||||||
- path: /etc/ssl/certs
|
- path: /etc/ssl/certs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
|
# The models and their one-off preparation, kept across restarts.
|
||||||
|
# Downloaded again if lost, so no backups: the no-backup group
|
||||||
|
# keeps it out of Longhorn's default one (backup-daily), and
|
||||||
|
# Velero skips that label too (velero/skip-emptydir-policy.yaml).
|
||||||
cache:
|
cache:
|
||||||
enabled: true
|
enabled: true
|
||||||
# Optional: to avoid downloading the ML models every start, use
|
type: persistentVolumeClaim
|
||||||
# a persistentVolumeClaim with size, accessMode and storageClass
|
storageClass: longhorn
|
||||||
type: emptyDir
|
accessMode: ReadWriteOnce
|
||||||
|
size: 10Gi
|
||||||
|
labels:
|
||||||
|
recurring-job.longhorn.io/source: enabled
|
||||||
|
recurring-job-group.longhorn.io/no-backup: enabled
|
||||||
|
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
|
|||||||
@@ -73,7 +73,9 @@ spec:
|
|||||||
---
|
---
|
||||||
# The CNPG operator checks each instance on its status port and connects to
|
# The CNPG operator checks each instance on its status port and connects to
|
||||||
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
||||||
# through the API server (the node, port 16443).
|
# through the API server (the node, port 16443). The barman cloud sidecar
|
||||||
|
# sends the WAL and the base backups to Garage on the NAS
|
||||||
|
# (postgresql-backup.yaml).
|
||||||
apiVersion: networking.k8s.io/v1
|
apiVersion: networking.k8s.io/v1
|
||||||
kind: NetworkPolicy
|
kind: NetworkPolicy
|
||||||
metadata:
|
metadata:
|
||||||
@@ -100,3 +102,34 @@ spec:
|
|||||||
cidr: 192.168.1.204/32
|
cidr: 192.168.1.204/32
|
||||||
ports:
|
ports:
|
||||||
- port: 16443
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.99/32
|
||||||
|
ports:
|
||||||
|
- port: 3900
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
# Backups of immich's database (albums, faces, the photos' metadata; the
|
||||||
|
# photos themselves are files on immich-claim) by CNPG itself, through the
|
||||||
|
# barman cloud plugin (apps/templates/plugin-barman-cloud-helm.yaml): a base
|
||||||
|
# backup every night, and every WAL segment as soon as it is full or 5
|
||||||
|
# minutes old (archive_timeout). Both go to the bucket cnpg-immich in Garage
|
||||||
|
# on the NAS (nas/garage), so the database can be restored to any moment of
|
||||||
|
# the last 30 days. Velero copies the data directory while PostgreSQL runs,
|
||||||
|
# which is not a consistent copy. Only the PostgreSQL 17 cluster; the 14
|
||||||
|
# one is about to go.
|
||||||
|
# The key is Garage's cnpg-immich, in Vault as kv/cnpg-immich-s3; it can
|
||||||
|
# only reach this bucket.
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: immich-postgres-17-s3
|
||||||
|
namespace: immich
|
||||||
|
spec:
|
||||||
|
secretStoreRef:
|
||||||
|
name: vault-backend
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
refreshInterval: 15m
|
||||||
|
target:
|
||||||
|
name: immich-postgres-17-s3
|
||||||
|
creationPolicy: Owner
|
||||||
|
template:
|
||||||
|
engineVersion: v2
|
||||||
|
data:
|
||||||
|
ACCESS_KEY_ID: "{{ .access_key_id }}"
|
||||||
|
ACCESS_SECRET_KEY: "{{ .secret_access_key }}"
|
||||||
|
# Garage rejects requests signed for another region
|
||||||
|
REGION: garage
|
||||||
|
data:
|
||||||
|
- secretKey: access_key_id
|
||||||
|
remoteRef:
|
||||||
|
key: kv/cnpg-immich-s3
|
||||||
|
property: access_key_id
|
||||||
|
- secretKey: secret_access_key
|
||||||
|
remoteRef:
|
||||||
|
key: kv/cnpg-immich-s3
|
||||||
|
property: secret_access_key
|
||||||
|
---
|
||||||
|
apiVersion: barmancloud.cnpg.io/v1
|
||||||
|
kind: ObjectStore
|
||||||
|
metadata:
|
||||||
|
name: immich-postgres-17
|
||||||
|
namespace: immich
|
||||||
|
annotations:
|
||||||
|
# The CRD comes with the plugin's own app
|
||||||
|
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||||
|
spec:
|
||||||
|
retentionPolicy: 30d
|
||||||
|
configuration:
|
||||||
|
destinationPath: s3://cnpg-immich/
|
||||||
|
endpointURL: http://192.168.1.99:3900
|
||||||
|
s3Credentials:
|
||||||
|
accessKeyId:
|
||||||
|
name: immich-postgres-17-s3
|
||||||
|
key: ACCESS_KEY_ID
|
||||||
|
secretAccessKey:
|
||||||
|
name: immich-postgres-17-s3
|
||||||
|
key: ACCESS_SECRET_KEY
|
||||||
|
region:
|
||||||
|
name: immich-postgres-17-s3
|
||||||
|
key: REGION
|
||||||
|
wal:
|
||||||
|
compression: gzip
|
||||||
|
data:
|
||||||
|
compression: gzip
|
||||||
|
instanceSidecarConfiguration:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
env:
|
||||||
|
# Newer boto3 adds checksums that S3-compatible stores can reject;
|
||||||
|
# the plugin's docs suggest sending them only when required.
|
||||||
|
- name: AWS_REQUEST_CHECKSUM_CALCULATION
|
||||||
|
value: when_required
|
||||||
|
- name: AWS_RESPONSE_CHECKSUM_VALIDATION
|
||||||
|
value: when_required
|
||||||
|
---
|
||||||
|
apiVersion: postgresql.cnpg.io/v1
|
||||||
|
kind: ScheduledBackup
|
||||||
|
metadata:
|
||||||
|
name: immich-postgres-17-daily
|
||||||
|
namespace: immich
|
||||||
|
spec:
|
||||||
|
cluster:
|
||||||
|
name: immich-postgres-17
|
||||||
|
# 03:00 UTC, after Longhorn's and Velero's backups; immediate: one right
|
||||||
|
# away, so WAL archiving has a base backup to start from.
|
||||||
|
schedule: "0 0 3 * * *"
|
||||||
|
immediate: true
|
||||||
|
backupOwnerReference: self
|
||||||
|
method: plugin
|
||||||
|
pluginConfiguration:
|
||||||
|
name: barman-cloud.cloudnative-pg.io
|
||||||
@@ -13,14 +13,26 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 20m
|
cpu: 20m
|
||||||
memory: 320Mi
|
memory: 320Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
|
|
||||||
postgresql:
|
postgresql:
|
||||||
shared_preload_libraries:
|
shared_preload_libraries:
|
||||||
- "vchord.so"
|
- "vchord.so"
|
||||||
|
|
||||||
|
# WAL archiving and base backups to Garage, see postgresql-backup.yaml
|
||||||
|
plugins:
|
||||||
|
- name: barman-cloud.cloudnative-pg.io
|
||||||
|
isWALArchiver: true
|
||||||
|
parameters:
|
||||||
|
barmanObjectName: immich-postgres-17
|
||||||
|
|
||||||
# Filled once from the PostgreSQL 14 cluster (immich-postgres) after its
|
# Filled once from the PostgreSQL 14 cluster (immich-postgres) after its
|
||||||
# embedding columns were turned into real[] and pgvecto.rs was dropped,
|
# embedding columns were turned into real[] and pgvecto.rs was dropped,
|
||||||
# see docs/upgrade-k8s-1.33.md.
|
# see docs/upgrade-k8s-1.33.md. That cluster is gone since 2026-10-28, so
|
||||||
|
# creating this Cluster again fails here, on purpose: an empty database
|
||||||
|
# would start immich with an empty library. Restore from Garage instead
|
||||||
|
# (bootstrap.recovery, see postgresql-backup.yaml).
|
||||||
bootstrap:
|
bootstrap:
|
||||||
initdb:
|
initdb:
|
||||||
database: immich
|
database: immich
|
||||||
|
|||||||
@@ -1,28 +0,0 @@
|
|||||||
apiVersion: postgresql.cnpg.io/v1
|
|
||||||
kind: Cluster
|
|
||||||
metadata:
|
|
||||||
name: immich-postgres
|
|
||||||
namespace: immich
|
|
||||||
spec:
|
|
||||||
instances: 1
|
|
||||||
|
|
||||||
# Use TensorChord's CNPG-compatible image with pgvecto.rs v0.2.1
|
|
||||||
imageName: ghcr.io/tensorchord/cloudnative-pgvecto.rs:14-v0.2.1
|
|
||||||
|
|
||||||
postgresql:
|
|
||||||
shared_preload_libraries:
|
|
||||||
- "vectors.so"
|
|
||||||
|
|
||||||
bootstrap:
|
|
||||||
initdb:
|
|
||||||
database: immich
|
|
||||||
owner: immich
|
|
||||||
postInitSQL:
|
|
||||||
- CREATE EXTENSION IF NOT EXISTS vectors;
|
|
||||||
- CREATE EXTENSION IF NOT EXISTS cube;
|
|
||||||
- CREATE EXTENSION IF NOT EXISTS earthdistance CASCADE;
|
|
||||||
- ALTER USER immich WITH SUPERUSER;
|
|
||||||
|
|
||||||
storage:
|
|
||||||
size: 20Gi
|
|
||||||
storageClass: longhorn
|
|
||||||
+66
-2
@@ -8,6 +8,10 @@
|
|||||||
# - the group account's library is shared as a partner and shown in
|
# - the group account's library is shared as a partner and shown in
|
||||||
# everyone's timeline, so search covers it (immich only searches your own
|
# everyone's timeline, so search covers it (immich only searches your own
|
||||||
# and your timeline partners' photos);
|
# and your timeline partners' photos);
|
||||||
|
# - everyone is put in the group account's cluster group and gets its
|
||||||
|
# people (person sharing, immich 3.3): people are per user, and a partner
|
||||||
|
# library doesn't bring its people along. Those who may upload can name
|
||||||
|
# them, for everyone;
|
||||||
# - the invitation notifications for albums older than a week are hidden,
|
# - the invitation notifications for albums older than a week are hidden,
|
||||||
# or a new member would get one per album;
|
# or a new member would get one per album;
|
||||||
# - sessions of OAuth users older than SESSION_MAX_AGE_DAYS are ended, so
|
# - sessions of OAuth users older than SESSION_MAX_AGE_DAYS are ended, so
|
||||||
@@ -15,8 +19,11 @@
|
|||||||
# Online.
|
# Online.
|
||||||
# Albums and partners go through the API. Joining other people's albums
|
# Albums and partners go through the API. Joining other people's albums
|
||||||
# (the API only lets an album's owner or editors share it), the timeline
|
# (the API only lets an album's owner or editors share it), the timeline
|
||||||
# flag (which only the receiving user can set), the notifications and the
|
# flag and the cluster group (which only the receiving user can set or
|
||||||
# sessions go straight to the database.
|
# accept), sharing the people (the API wants person.update, which the
|
||||||
|
# group's key lacks, and with password login off nobody can log in as the
|
||||||
|
# group to add it), the notifications and the sessions go straight to the
|
||||||
|
# database.
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
metadata:
|
metadata:
|
||||||
@@ -24,6 +31,7 @@ metadata:
|
|||||||
namespace: immich
|
namespace: immich
|
||||||
data:
|
data:
|
||||||
sync.py: |
|
sync.py: |
|
||||||
|
import collections
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -129,6 +137,62 @@ data:
|
|||||||
'UPDATE partner SET "inTimeline" = true WHERE "sharedById" = :\'group\' AND NOT "inTimeline"',
|
'UPDATE partner SET "inTimeline" = true WHERE "sharedById" = :\'group\' AND NOT "inTimeline"',
|
||||||
group=group_id,
|
group=group_id,
|
||||||
)
|
)
|
||||||
|
# What accepting an invitation to the group account's cluster group does
|
||||||
|
# (ClusterGroupService.acceptRequest): the person groups only the member
|
||||||
|
# has people in move along. Members with people in groups that others
|
||||||
|
# also have people in are left alone; that only happens after sharing.
|
||||||
|
for (email,) in sql(
|
||||||
|
"""
|
||||||
|
BEGIN;
|
||||||
|
CREATE TEMP TABLE moving ON COMMIT DROP AS
|
||||||
|
SELECT u.id FROM "user" u, "user" g
|
||||||
|
WHERE g.id = :'group' AND u.id = ANY (:'members'::uuid[])
|
||||||
|
AND u."clusterGroupId" <> g."clusterGroupId"
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM person p
|
||||||
|
JOIN person o ON o."personGroupId" = p."personGroupId" AND o."ownerId" <> p."ownerId"
|
||||||
|
WHERE p."ownerId" = u.id
|
||||||
|
);
|
||||||
|
UPDATE person_group SET "clusterGroupId" = (SELECT "clusterGroupId" FROM "user" WHERE id = :'group')
|
||||||
|
WHERE id IN (SELECT p."personGroupId" FROM person p JOIN moving m ON m.id = p."ownerId");
|
||||||
|
DELETE FROM cluster_group_request
|
||||||
|
WHERE "userId" IN (SELECT id FROM moving)
|
||||||
|
AND "clusterGroupId" = (SELECT "clusterGroupId" FROM "user" WHERE id = :'group');
|
||||||
|
UPDATE "user" SET "clusterGroupId" = (SELECT "clusterGroupId" FROM "user" WHERE id = :'group')
|
||||||
|
WHERE id IN (SELECT id FROM moving)
|
||||||
|
RETURNING email;
|
||||||
|
COMMIT;
|
||||||
|
""",
|
||||||
|
group=group_id,
|
||||||
|
members="{" + ",".join(members) + "}",
|
||||||
|
):
|
||||||
|
print(f"put {email} in the group's cluster group", flush=True)
|
||||||
|
# What PUT /people/users does for the group account: its people, with
|
||||||
|
# their names, in every member's people list and search filters. Write
|
||||||
|
# access (naming people, for everyone) for those who may upload.
|
||||||
|
shared = sql(
|
||||||
|
"""
|
||||||
|
INSERT INTO person_user ("personGroupId", "sharedById", "sharedWithId", role)
|
||||||
|
SELECT p."personGroupId", p."ownerId", m.id, m.role::person_user_role_enum
|
||||||
|
FROM person p
|
||||||
|
JOIN "user" g ON g.id = p."ownerId"
|
||||||
|
CROSS JOIN unnest(:'ids'::uuid[], :'roles'::text[]) AS m(id, role)
|
||||||
|
JOIN "user" u ON u.id = m.id AND u."clusterGroupId" = g."clusterGroupId"
|
||||||
|
WHERE p."ownerId" = :'group' AND NOT p."isHidden"
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM person_user pu
|
||||||
|
WHERE pu."personGroupId" = p."personGroupId" AND pu."sharedById" = p."ownerId"
|
||||||
|
AND pu."sharedWithId" = m.id AND pu.role = m.role::person_user_role_enum
|
||||||
|
)
|
||||||
|
ON CONFLICT ("personGroupId", "sharedById", "sharedWithId") DO UPDATE SET role = EXCLUDED.role
|
||||||
|
RETURNING "sharedWithId", role
|
||||||
|
""",
|
||||||
|
group=group_id,
|
||||||
|
ids="{" + ",".join(wanted) + "}",
|
||||||
|
roles="{" + ",".join("read" if role == "viewer" else "write" for role in wanted.values()) + "}",
|
||||||
|
)
|
||||||
|
for (uid, role), count in sorted(collections.Counter(map(tuple, shared)).items()):
|
||||||
|
print(f"share {count} people with {members[uid]} ({role})", flush=True)
|
||||||
sql(
|
sql(
|
||||||
"""
|
"""
|
||||||
UPDATE notification SET "deletedAt" = now()
|
UPDATE notification SET "deletedAt" = now()
|
||||||
|
|||||||
@@ -3,3 +3,4 @@ kind: Kustomization
|
|||||||
resources:
|
resources:
|
||||||
- operator-crds-helm.yaml
|
- operator-crds-helm.yaml
|
||||||
- operator-helm.yaml
|
- operator-helm.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
# The mariadb-operator's three pods reach each other, DNS and the API
|
||||||
|
# server (the node, port 16443); the operator also connects to MariaDB on
|
||||||
|
# 3306 for the databases, users and grants it manages (mbgwp's
|
||||||
|
# mariadb-from-operator lets it in). Besides their own namespace they take
|
||||||
|
# connections only from the node: the API server calls the webhook from
|
||||||
|
# there and the kubelet probes come from there too (10.1.7.64 is the node's
|
||||||
|
# Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr). This
|
||||||
|
# folder's Application deploys into argocd (the operator's own
|
||||||
|
# Applications), hence the namespace here.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
namespace: mariadb-system
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: mariadb
|
||||||
|
ports:
|
||||||
|
- port: 3306
|
||||||
@@ -13,7 +13,7 @@ spec:
|
|||||||
source:
|
source:
|
||||||
chart: mariadb-operator-crds
|
chart: mariadb-operator-crds
|
||||||
repoURL: https://mariadb-operator.github.io/mariadb-operator
|
repoURL: https://mariadb-operator.github.io/mariadb-operator
|
||||||
targetRevision: 25.10.4
|
targetRevision: 26.10.2
|
||||||
syncPolicy:
|
syncPolicy:
|
||||||
automated:
|
automated:
|
||||||
prune: true
|
prune: true
|
||||||
|
|||||||
@@ -13,13 +13,15 @@ spec:
|
|||||||
source:
|
source:
|
||||||
chart: mariadb-operator
|
chart: mariadb-operator
|
||||||
repoURL: https://mariadb-operator.github.io/mariadb-operator
|
repoURL: https://mariadb-operator.github.io/mariadb-operator
|
||||||
targetRevision: 25.10.4
|
targetRevision: 26.10.2
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 48Mi
|
memory: 48Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
metrics:
|
metrics:
|
||||||
enabled: true
|
enabled: true
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
@@ -29,6 +31,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 32Mi
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: false
|
enabled: false
|
||||||
certController:
|
certController:
|
||||||
@@ -36,6 +40,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 32Mi
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: false
|
enabled: false
|
||||||
syncPolicy:
|
syncPolicy:
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ spec:
|
|||||||
name: mbgwp-mariadb-credentials
|
name: mbgwp-mariadb-credentials
|
||||||
key: root-password
|
key: root-password
|
||||||
|
|
||||||
image: mariadb:11.4.5
|
image: mariadb:11.4.13
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
|
|
||||||
port: 3306
|
port: 3306
|
||||||
|
|||||||
@@ -85,3 +85,29 @@ spec:
|
|||||||
kubernetes.io/metadata.name: mariadb-system
|
kubernetes.io/metadata.name: mariadb-system
|
||||||
ports:
|
ports:
|
||||||
- port: 3306
|
- port: 3306
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# Garage on the NAS
|
||||||
|
|
||||||
|
S3 storage for Velero, on the Synology (192.168.1.99) in Docker, outside
|
||||||
|
the cluster. Replaced MinIO, which is no longer maintained. One node, data
|
||||||
|
in `/volume1/docker/garage/{meta,data}`. S3 API on port 3900, admin API on
|
||||||
|
3903, both LAN only. Set up 2026-10-25 with Garage v2.5.0.
|
||||||
|
|
||||||
|
The NAS is reached as `ssh -p 34000 rubenhensen@192.168.1.99`; docker is
|
||||||
|
in `/usr/local/bin`, which a non-interactive SSH session doesn't have on
|
||||||
|
its PATH.
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
`garage.toml` here has placeholders for the two secrets; on the NAS they
|
||||||
|
were filled in once with `openssl rand -hex 32` (`rpc_secret`) and
|
||||||
|
`openssl rand -base64 32` (`admin_token`), and only exist there.
|
||||||
|
|
||||||
|
```
|
||||||
|
cd /volume1/docker/garage
|
||||||
|
docker compose up -d
|
||||||
|
g="docker exec garage /garage"
|
||||||
|
$g layout assign -z nas -c 800G <node id from $g status>
|
||||||
|
$g layout apply --version 1
|
||||||
|
$g bucket create velero-backups
|
||||||
|
$g key create velero
|
||||||
|
$g bucket allow --read --write --key velero velero-backups
|
||||||
|
```
|
||||||
|
|
||||||
|
Velero's key is in Vault as `kv/velero-s3` (`access_key_id`,
|
||||||
|
`secret_access_key`) and reaches Velero through
|
||||||
|
`velero/external-secret.yaml`. Without the cluster, `$g key info velero
|
||||||
|
--show-secret` on the NAS shows it again.
|
||||||
|
|
||||||
|
CNPG's backups (`authentik/postgresql-backup.yaml`,
|
||||||
|
`immich/postgresql-backup.yaml`) have a bucket and a key per cluster, so
|
||||||
|
neither database can touch the other's backups. Made on 2026-10-26 the
|
||||||
|
same way as Velero's, with `cnpg-authentik` and `cnpg-immich` as both
|
||||||
|
bucket and key name; the keys are in Vault as `kv/cnpg-authentik-s3` and
|
||||||
|
`kv/cnpg-immich-s3`.
|
||||||
|
|
||||||
|
Renovate updates the image tag in `compose.yaml`, but nothing deploys it:
|
||||||
|
after merging, `docker compose pull && docker compose up -d` on the NAS.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
services:
|
||||||
|
garage:
|
||||||
|
image: dxflrs/garage:v2.5.0
|
||||||
|
container_name: garage
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "3900:3900" # S3 API
|
||||||
|
- "3903:3903" # admin API
|
||||||
|
volumes:
|
||||||
|
- ./garage.toml:/etc/garage.toml:ro
|
||||||
|
- ./meta:/var/lib/garage/meta
|
||||||
|
- ./data:/var/lib/garage/data
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# One node on the Synology. Velero's kopia data is encrypted, so
|
||||||
|
# compression would only cost CPU.
|
||||||
|
#
|
||||||
|
# SQLite rather than the default LMDB: LMDB can be corrupted by an unclean
|
||||||
|
# shutdown, and a single node has no other node to repair from. With
|
||||||
|
# metadata_fsync SQLite runs WAL with synchronous=NORMAL, which only syncs
|
||||||
|
# at checkpoints; LMDB with metadata_fsync synced every transaction, ~270
|
||||||
|
# writes/s on the NAS's two disks with Garage idle. Converted 2026-10-25
|
||||||
|
# with `garage convert-db`.
|
||||||
|
metadata_dir = "/var/lib/garage/meta"
|
||||||
|
data_dir = "/var/lib/garage/data"
|
||||||
|
db_engine = "sqlite"
|
||||||
|
metadata_fsync = true
|
||||||
|
data_fsync = true
|
||||||
|
metadata_auto_snapshot_interval = "6h"
|
||||||
|
replication_factor = 1
|
||||||
|
compression_level = "none"
|
||||||
|
|
||||||
|
rpc_bind_addr = "[::]:3901"
|
||||||
|
rpc_public_addr = "127.0.0.1:3901"
|
||||||
|
rpc_secret = "@RPC_SECRET@"
|
||||||
|
|
||||||
|
[s3_api]
|
||||||
|
s3_region = "garage"
|
||||||
|
api_bind_addr = "[::]:3900"
|
||||||
|
|
||||||
|
[admin]
|
||||||
|
api_bind_addr = "[::]:3903"
|
||||||
|
admin_token = "@ADMIN_TOKEN@"
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
let
|
let
|
||||||
# Public IP of the home network where the k8s cluster ingress lives.
|
# Public IP of the home network where the k8s cluster ingress lives.
|
||||||
# Keep in sync with dns/domains/rubenhensen.nl.yaml.
|
# Keep in sync with dns/domains/rubenhensen.nl.yaml.
|
||||||
homeIP = "62.41.86.27";
|
homeIP = "62.41.86.53";
|
||||||
|
|
||||||
# Hosts tunneled to the home k8s cluster. Each entry gets:
|
# Hosts tunneled to the home k8s cluster. Each entry gets:
|
||||||
# * an SNI map entry for TCP passthrough on :443
|
# * an SNI map entry for TCP passthrough on :443
|
||||||
@@ -173,12 +173,18 @@ in
|
|||||||
# - mail.rubenhensen.nl (and anything else) → local stalwart on 8443
|
# - mail.rubenhensen.nl (and anything else) → local stalwart on 8443
|
||||||
# - tunneled hosts → home cluster ingress on 443
|
# - tunneled hosts → home cluster ingress on 443
|
||||||
# - s1 / ntfy1.rubenhensen.nl → local nginx on 8444 (Gatus, ntfy)
|
# - s1 / ntfy1.rubenhensen.nl → local nginx on 8444 (Gatus, ntfy)
|
||||||
|
#
|
||||||
|
# Each connection starts with a PROXY header that carries the visitor's
|
||||||
|
# address, so the home cluster sees who is asking instead of this box
|
||||||
|
# (gateway/proxy-protocol.yaml in k8scd). nginx sends the header to
|
||||||
|
# every upstream or none, so the local ones go through a hop on
|
||||||
|
# 8445/8446 that takes it off again.
|
||||||
streamConfig = ''
|
streamConfig = ''
|
||||||
map $ssl_preread_server_name $tunnel_upstream {
|
map $ssl_preread_server_name $tunnel_upstream {
|
||||||
${sniMapEntries}
|
${sniMapEntries}
|
||||||
s1.rubenhensen.nl 127.0.0.1:8444;
|
s1.rubenhensen.nl 127.0.0.1:8446;
|
||||||
ntfy1.rubenhensen.nl 127.0.0.1:8444;
|
ntfy1.rubenhensen.nl 127.0.0.1:8446;
|
||||||
default 127.0.0.1:8443;
|
default 127.0.0.1:8445;
|
||||||
}
|
}
|
||||||
|
|
||||||
server {
|
server {
|
||||||
@@ -186,6 +192,17 @@ ${sniMapEntries}
|
|||||||
listen [::]:443;
|
listen [::]:443;
|
||||||
proxy_pass $tunnel_upstream;
|
proxy_pass $tunnel_upstream;
|
||||||
ssl_preread on;
|
ssl_preread on;
|
||||||
|
proxy_protocol on;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 127.0.0.1:8445 proxy_protocol;
|
||||||
|
proxy_pass 127.0.0.1:8443;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 127.0.0.1:8446 proxy_protocol;
|
||||||
|
proxy_pass 127.0.0.1:8444;
|
||||||
}
|
}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
|||||||
@@ -65,3 +65,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: ntfy
|
- name: ntfy
|
||||||
image: binwiederhier/ntfy:v2.28.0
|
image: binwiederhier/ntfy:v2.37.2
|
||||||
args: ["serve"]
|
args: ["serve"]
|
||||||
env:
|
env:
|
||||||
- name: NTFY_BASE_URL
|
- name: NTFY_BASE_URL
|
||||||
|
|||||||
+48
-4
@@ -4,9 +4,18 @@
|
|||||||
"config:recommended",
|
"config:recommended",
|
||||||
"customManagers:githubActionsVersions"
|
"customManagers:githubActionsVersions"
|
||||||
],
|
],
|
||||||
|
"timezone": "Europe/Amsterdam",
|
||||||
"argocd": {
|
"argocd": {
|
||||||
|
"description": "Every app folder; Renovate's regexes are RE2, which has no lookahead, so the other folders are skipped through ignorePaths",
|
||||||
"managerFilePatterns": [
|
"managerFilePatterns": [
|
||||||
"/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
|
"/^[^/]+/.+\\.yaml$/"
|
||||||
|
],
|
||||||
|
"ignorePaths": [
|
||||||
|
"nix-infra-machine/**",
|
||||||
|
"backup-testing/**",
|
||||||
|
"docs/**",
|
||||||
|
"tools/**",
|
||||||
|
".github/**"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"helm-values": {
|
"helm-values": {
|
||||||
@@ -15,8 +24,16 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"kubernetes": {
|
"kubernetes": {
|
||||||
|
"description": "Every app folder, as for argocd",
|
||||||
"managerFilePatterns": [
|
"managerFilePatterns": [
|
||||||
"/^(?!(nix-infra-machine|backup-testing|docs|tools|\\.github)/)[^/]+/.+\\.yaml$/"
|
"/^[^/]+/.+\\.yaml$/"
|
||||||
|
],
|
||||||
|
"ignorePaths": [
|
||||||
|
"nix-infra-machine/**",
|
||||||
|
"backup-testing/**",
|
||||||
|
"docs/**",
|
||||||
|
"tools/**",
|
||||||
|
".github/**"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"customManagers": [
|
"customManagers": [
|
||||||
@@ -65,13 +82,40 @@
|
|||||||
"description": "A PostgreSQL major upgrade needs pg_upgrade or a dump, not a tag bump",
|
"description": "A PostgreSQL major upgrade needs pg_upgrade or a dump, not a tag bump",
|
||||||
"matchDepNames": [
|
"matchDepNames": [
|
||||||
"ghcr.io/cloudnative-pg/postgresql",
|
"ghcr.io/cloudnative-pg/postgresql",
|
||||||
"ghcr.io/tensorchord/cloudnative-vectorchord",
|
"ghcr.io/tensorchord/cloudnative-vectorchord"
|
||||||
"ghcr.io/tensorchord/cloudnative-pgvecto.rs"
|
|
||||||
],
|
],
|
||||||
"matchUpdateTypes": [
|
"matchUpdateTypes": [
|
||||||
"major"
|
"major"
|
||||||
],
|
],
|
||||||
"enabled": false
|
"enabled": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "MariaDB goes from one LTS to the next by hand, after a dump: there is no way back. 11.4 has community support until May 2029, longer than 11.8 (June 2028); the next step is 12.3 (June 2029). Within 11.4 the patches come through",
|
||||||
|
"matchDepNames": [
|
||||||
|
"mariadb"
|
||||||
|
],
|
||||||
|
"allowedVersions": "<11.5"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Patch updates of apps that hold little state and are easy to roll back, and of CI's tools, merge themselves once CI passes: three days after the release, in the evening or at the weekend, when the cluster can be reached. Not the databases, operators, Vault, authentik, Longhorn, Velero, immich, Gitea or WordPress. Renovate merges itself; the repo has no branch protection for GitHub's auto-merge to wait on",
|
||||||
|
"matchFileNames": [
|
||||||
|
"ntfy/**",
|
||||||
|
"gatus/**",
|
||||||
|
"freshrss/**",
|
||||||
|
"serpbear/**",
|
||||||
|
"actual-budget/**",
|
||||||
|
".github/workflows/**"
|
||||||
|
],
|
||||||
|
"matchUpdateTypes": [
|
||||||
|
"patch"
|
||||||
|
],
|
||||||
|
"minimumReleaseAge": "3 days",
|
||||||
|
"automerge": true,
|
||||||
|
"platformAutomerge": false,
|
||||||
|
"automergeSchedule": [
|
||||||
|
"* 17-21 * * 1-5",
|
||||||
|
"* 10-21 * * 0,6"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -54,3 +54,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -51,7 +51,7 @@ spec:
|
|||||||
value: --use-openssl-ca
|
value: --use-openssl-ca
|
||||||
- name: SSL_CERT_FILE
|
- name: SSL_CERT_FILE
|
||||||
value: /etc/ssl/certs/ca-certificates.crt
|
value: /etc/ssl/certs/ca-certificates.crt
|
||||||
image: towfiqi/serpbear:3.1.0
|
image: towfiqi/serpbear:3.1.2
|
||||||
name: serpbear
|
name: serpbear
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 3000
|
- containerPort: 3000
|
||||||
@@ -60,6 +60,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 160Mi
|
memory: 160Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
readinessProbe:
|
readinessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /login
|
path: /login
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module dqlite-vacuum
|
|||||||
go 1.27.3
|
go 1.27.3
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.1
|
github.com/canonical/go-dqlite/v3 v3.0.4
|
||||||
gopkg.in/yaml.v2 v2.4.0
|
gopkg.in/yaml.v2 v2.4.0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
|
|||||||
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
||||||
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
||||||
|
github.com/canonical/go-dqlite/v3 v3.0.4 h1:JhITsrvVL0pii9arxDksXfsRqBI+je1VpcqVuzDllUM=
|
||||||
|
github.com/canonical/go-dqlite/v3 v3.0.4/go.mod h1:6O+E6MiYesijRUOECyywp16iT+N4QOrvjDWWiMJ/xF0=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@@ -26,7 +29,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||||
|
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
|||||||
@@ -66,3 +66,29 @@ spec:
|
|||||||
- 169.254.0.0/16
|
- 169.254.0.0/16
|
||||||
- 172.16.0.0/12
|
- 172.16.0.0/12
|
||||||
- 192.168.0.0/16
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# cert-manager's HTTP-01 solver, which runs only while a certificate
|
||||||
|
# renews: Let's Encrypt and cert-manager's own self check reach it on 8089
|
||||||
|
# through Envoy. The policies above let Envoy reach the app's port only,
|
||||||
|
# which left the solver unreachable (found 2026-10-26, when ynab's renewal
|
||||||
|
# hung).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: acme-http01-solver
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
acme.cert-manager.io/http01-solver: "true"
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8089
|
||||||
@@ -7,6 +7,8 @@ injector:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 32Mi
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
# The chart's default anti-affinity forbids two injector pods on one
|
# The chart's default anti-affinity forbids two injector pods on one
|
||||||
# node, so on this single node a rolling update can never start the new
|
# node, so on this single node a rolling update can never start the new
|
||||||
# pod while the old one runs. Recreate removes the old pod first, which
|
# pod while the old one runs. Recreate removes the old pod first, which
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# Velero's key for its bucket in Garage on the NAS (nas/garage), in the
|
||||||
|
# AWS credentials file format the aws plugin and kopia read.
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: velero-s3-credentials
|
||||||
|
spec:
|
||||||
|
secretStoreRef:
|
||||||
|
name: vault-backend
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
refreshInterval: 15m
|
||||||
|
target:
|
||||||
|
name: velero-s3-credentials
|
||||||
|
creationPolicy: Owner
|
||||||
|
template:
|
||||||
|
engineVersion: v2
|
||||||
|
data:
|
||||||
|
cloud: |
|
||||||
|
[default]
|
||||||
|
aws_access_key_id={{ .access_key_id }}
|
||||||
|
aws_secret_access_key={{ .secret_access_key }}
|
||||||
|
data:
|
||||||
|
- secretKey: access_key_id
|
||||||
|
remoteRef:
|
||||||
|
key: kv/velero-s3
|
||||||
|
property: access_key_id
|
||||||
|
- secretKey: secret_access_key
|
||||||
|
remoteRef:
|
||||||
|
key: kv/velero-s3
|
||||||
|
property: secret_access_key
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: nas-minio-external
|
|
||||||
spec:
|
|
||||||
type: ExternalName
|
|
||||||
externalName: 192.168.1.99
|
|
||||||
ports:
|
|
||||||
- port: 9000
|
|
||||||
protocol: TCP
|
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Velero, its node agent, and the pods it starts here (the data movers of
|
||||||
|
# each pod volume backup or restore, kopia's maintenance jobs, the chart's
|
||||||
|
# CRD upgrade job) reach each other, DNS, the API server (the node, port
|
||||||
|
# 16443) and Garage on the NAS (192.168.1.99:3900, nas/garage). The node
|
||||||
|
# agent reads volumes through its hostPath mount, not the network. Besides
|
||||||
|
# their own namespace they take connections only from the node (kubelet
|
||||||
|
# probes; 10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr); nothing scrapes the metrics on
|
||||||
|
# 8085. MinIO (the NAS, port 9000) is not allowed: going back to it means
|
||||||
|
# adding its port here as well.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.99/32
|
||||||
|
ports:
|
||||||
|
- port: 3900
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
# Volume policy for the backup schedules. emptyDir volumes hold nothing
|
# Volume policy for the backup schedules. emptyDir volumes hold nothing
|
||||||
# worth keeping, and backing them up fails when the pod is gone before its
|
# worth keeping, and backing them up fails when the pod is gone before its
|
||||||
# volume is read, as happens with Velero's own repository maintenance jobs.
|
# volume is read, as happens with Velero's own repository maintenance jobs.
|
||||||
|
# PVCs in Longhorn's no-backup group (caches, such as immich's ML models)
|
||||||
|
# are skipped too.
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
metadata:
|
metadata:
|
||||||
@@ -14,3 +16,8 @@ data:
|
|||||||
- emptyDir
|
- emptyDir
|
||||||
action:
|
action:
|
||||||
type: skip
|
type: skip
|
||||||
|
- conditions:
|
||||||
|
pvcLabels:
|
||||||
|
recurring-job-group.longhorn.io/no-backup: enabled
|
||||||
|
action:
|
||||||
|
type: skip
|
||||||
+11
-10
@@ -8,7 +8,7 @@ spec:
|
|||||||
source:
|
source:
|
||||||
chart: velero
|
chart: velero
|
||||||
repoURL: https://vmware-tanzu.github.io/helm-charts
|
repoURL: https://vmware-tanzu.github.io/helm-charts
|
||||||
targetRevision: 12.2.1
|
targetRevision: 12.2.3
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
# Velero builds its repository maintenance jobs from a Deployment
|
# Velero builds its repository maintenance jobs from a Deployment
|
||||||
@@ -27,20 +27,22 @@ spec:
|
|||||||
- mountPath: /target
|
- mountPath: /target
|
||||||
name: plugins
|
name: plugins
|
||||||
configuration:
|
configuration:
|
||||||
|
# Garage on the NAS (nas/garage). Until 2026-10-26 this was MinIO on
|
||||||
|
# the same NAS; the bucket was copied over with rclone, so the
|
||||||
|
# existing backups and kopia repositories carried on.
|
||||||
backupStorageLocation:
|
backupStorageLocation:
|
||||||
- name: default
|
- name: default
|
||||||
bucket: velero-backups
|
bucket: velero-backups
|
||||||
provider: aws
|
provider: aws
|
||||||
config:
|
config:
|
||||||
region: minio
|
region: garage
|
||||||
s3ForcePathStyle: true
|
s3ForcePathStyle: true
|
||||||
s3Url: http://192.168.1.99:9000
|
s3Url: http://192.168.1.99:3900
|
||||||
insecureSkipTLSVerify: "true"
|
|
||||||
volumeSnapshotLocation:
|
volumeSnapshotLocation:
|
||||||
- name: default
|
- name: default
|
||||||
provider: aws
|
provider: aws
|
||||||
config:
|
config:
|
||||||
region: minio
|
region: garage
|
||||||
features: EnableCSI
|
features: EnableCSI
|
||||||
# Kopia maintenance ran every hour for each of the 15 repositories,
|
# Kopia maintenance ran every hour for each of the 15 repositories,
|
||||||
# and once a day as a full run of several minutes that writes its
|
# and once a day as a full run of several minutes that writes its
|
||||||
@@ -59,11 +61,8 @@ spec:
|
|||||||
memoryRequest: 256Mi
|
memoryRequest: 256Mi
|
||||||
memoryLimit: 2Gi
|
memoryLimit: 2Gi
|
||||||
credentials:
|
credentials:
|
||||||
# Use existing secret created by External Secrets Operator
|
# From Vault, velero/external-secret.yaml
|
||||||
# existingSecret: velero-minio-credentials
|
existingSecret: velero-s3-credentials
|
||||||
# extraEnvVars:
|
|
||||||
# AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
|
|
||||||
existingSecret: cloud-credentials
|
|
||||||
deployNodeAgent: true
|
deployNodeAgent: true
|
||||||
nodeAgent:
|
nodeAgent:
|
||||||
containerSecurityContext:
|
containerSecurityContext:
|
||||||
@@ -72,6 +71,8 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
cpu: 10m
|
cpu: 10m
|
||||||
memory: 64Mi
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
# 512Mi got the server OOMKilled 314 times, including halfway
|
# 512Mi got the server OOMKilled 314 times, including halfway
|
||||||
# through a backup of ~14.7k items.
|
# through a backup of ~14.7k items.
|
||||||
resources:
|
resources:
|
||||||
|
|||||||
Reference in new issue
Block a user