mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-15 22:29:09 +01:00
Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c384ae921f | ||
|
|
13b92fff54 | ||
|
|
3834f7a354 | ||
|
|
8110a235a9 | ||
|
|
fa33bf44ae | ||
|
|
d4e99d28c7 | ||
|
|
c8a81e805d | ||
|
|
0dc69d34f0 | ||
|
|
b6462fb3a9 | ||
|
|
bed68c70a8 | ||
|
|
3991a8d726 | ||
|
|
648f99c505 | ||
|
|
bdafcaf1e0 | ||
|
|
03c3196242 |
No files matched your search
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: cloudnative-pg-folder
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
|
targetRevision: HEAD
|
||||||
|
path: cloudnative-pg
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: cnpg-system
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
selfHeal: true
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: default-folder
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
|
targetRevision: HEAD
|
||||||
|
path: default
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: default
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
selfHeal: true
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: external-secrets-folder
|
||||||
|
namespace: argocd
|
||||||
|
finalizers:
|
||||||
|
- resources-finalizer.argocd.argoproj.io
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
|
targetRevision: HEAD
|
||||||
|
path: external-secrets
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: external-secrets
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
selfHeal: true
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
|
# k3s in Docker, to prove the backups can be restored. Nothing on the NAS
|
||||||
# changes:
|
# changes:
|
||||||
# - one backup (backups/<name>/) and the kopia repositories of the chosen
|
# - one backup (backups/<name>/) and the kopia repositories of the chosen
|
||||||
# namespaces (kopia/<namespace>/) are copied read-only from MinIO with
|
# namespaces (kopia/<namespace>/) are copied read-only from the NAS with
|
||||||
# rclone into a Docker volume. A second Velero on the real bucket could
|
# rclone into a Docker volume. A second Velero on the real bucket could
|
||||||
# run kopia maintenance on the same repositories as the cluster's own
|
# run kopia maintenance on the same repositories as the cluster's own
|
||||||
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
|
# - `rclone serve s3` serves that copy to a Velero 1.18.2 in k3s, with a
|
||||||
@@ -19,7 +19,7 @@
|
|||||||
# Defaults: the newest Completed daily-backup, and ntfy freshrss
|
# Defaults: the newest Completed daily-backup, and ntfy freshrss
|
||||||
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
|
# actual-budget. Not gatus: a restored Gatus would start sending alerts.
|
||||||
#
|
#
|
||||||
# It reads Velero's MinIO key from the cluster (velero/cloud-credentials)
|
# It reads Velero's S3 key from the cluster (velero/velero-s3-credentials)
|
||||||
# into a temporary directory, and the copied backup holds every Secret of
|
# into a temporary directory, and the copied backup holds every Secret of
|
||||||
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
|
# the cluster; all of it is deleted at the end. KEEP=1 keeps the test
|
||||||
# cluster (kubeconfig in the printed directory) until you run it with
|
# cluster (kubeconfig in the printed directory) until you run it with
|
||||||
@@ -71,14 +71,14 @@ wait_for() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
# Endpoint, region and bucket as Velero itself uses them. MinIO answers a
|
# Endpoint, region and bucket as Velero itself uses them. The NAS rejects a
|
||||||
# request signed for the wrong region (rclone defaults to us-east-1) with a
|
# request signed for the wrong region (rclone defaults to us-east-1); MinIO
|
||||||
# bare 400 on HEAD.
|
# did so with a bare 400 on HEAD.
|
||||||
if [[ -n ${FROM_VOLUME:-} ]]; then
|
if [[ -n ${FROM_VOLUME:-} ]]; then
|
||||||
BUCKET=${BUCKET:-velero-backups}
|
BUCKET=${BUCKET:-velero-backups}
|
||||||
else
|
else
|
||||||
bsl=$(pk -n velero get backupstoragelocations default -o json)
|
bsl=$(pk -n velero get backupstoragelocations default -o json)
|
||||||
MINIO=${MINIO:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
|
S3URL=${S3URL:-$(jq -r '.spec.config.s3Url' <<<"$bsl")}
|
||||||
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
|
REGION=${REGION:-$(jq -r '.spec.config.region // "us-east-1"' <<<"$bsl")}
|
||||||
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
|
BUCKET=${BUCKET:-$(jq -r '.spec.objectStorage.bucket' <<<"$bsl")}
|
||||||
fi
|
fi
|
||||||
@@ -101,10 +101,10 @@ if [[ -n ${FROM_VOLUME:-} ]]; then
|
|||||||
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
|
docker run --rm -v "$FROM_VOLUME:/from:ro" -v "$VOL:/data" alpine:3.22 cp -a /from/. /data/
|
||||||
else
|
else
|
||||||
echo "== copy from the NAS (read-only)"
|
echo "== copy from the NAS (read-only)"
|
||||||
(umask 077; pk -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
(umask 077; pk -n velero get secret velero-s3-credentials -o jsonpath='{.data.cloud}' | base64 -d |
|
||||||
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
awk -F' *= *' '/aws_access_key_id/ {print "RCLONE_CONFIG_NAS_ACCESS_KEY_ID="$2}
|
||||||
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
/aws_secret_access_key/ {print "RCLONE_CONFIG_NAS_SECRET_ACCESS_KEY="$2}' > "$WORK/nas.env"
|
||||||
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Minio\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$MINIO" "$REGION" >> "$WORK/nas.env")
|
printf 'RCLONE_CONFIG_NAS_TYPE=s3\nRCLONE_CONFIG_NAS_PROVIDER=Other\nRCLONE_CONFIG_NAS_ENDPOINT=%s\nRCLONE_CONFIG_NAS_REGION=%s\n' "$S3URL" "$REGION" >> "$WORK/nas.env")
|
||||||
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
copy() { docker run --rm --env-file "$WORK/nas.env" -v "$VOL:/data" "$RCLONE" copy --stats-one-line "nas:$BUCKET/$1" "/data/$BUCKET/$1"; }
|
||||||
copy "backups/$backup"
|
copy "backups/$backup"
|
||||||
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
for ns in "${namespaces[@]}"; do copy "kopia/$ns"; done
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# cert-manager's three pods reach each other, DNS and the API server (the
|
||||||
|
# node, port 16443), and take connections only from each other and the
|
||||||
|
# node: the API server calls the webhook from there and the kubelet probes
|
||||||
|
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). Nothing scrapes the metrics on
|
||||||
|
# 9402. The HTTP-01 solver pods run in each Certificate's own namespace
|
||||||
|
# (acme-http01-solver in <namespace>/networkpolicy.yaml), not here.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
---
|
||||||
|
# The controller talks to Let's Encrypt (443), and before it lets Let's
|
||||||
|
# Encrypt check a challenge it fetches http://<host>/.well-known/... itself
|
||||||
|
# (80). Every certificate's host has a public address, node001's or the
|
||||||
|
# home router's (checked 2026-10-26), so the LAN stays out.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: controller-to-internet
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: cert-manager
|
||||||
|
app.kubernetes.io/component: controller
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
- port: 443
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# The CloudNativePG operator reaches DNS, the API server (the node, port
|
||||||
|
# 16443) and the PostgreSQL instances it runs in other namespaces: the
|
||||||
|
# instance manager's status port (8000) and PostgreSQL (5432), which the
|
||||||
|
# instances' own policies let in (postgres-cnpg in authentik and immich).
|
||||||
|
# Besides its own namespace it takes connections only from the node: the
|
||||||
|
# API server calls its webhook (9443) from there and the kubelet probes
|
||||||
|
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). The chart's Application
|
||||||
|
# (apps/templates/cloudnativepg-helm.yaml) only installs the chart, so
|
||||||
|
# this folder gets an Application of its own, cloudnative-pg-folder.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
podSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: cnpg.io/cluster
|
||||||
|
operator: Exists
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
- port: 5432
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# Nothing runs in default (2026-10-26: only the kubernetes Service, whose
|
||||||
|
# endpoint is the API server on the node, which this doesn't touch). A pod
|
||||||
|
# started here, by `kubectl run` or a manifest without a namespace, gets
|
||||||
|
# no network at all, instead of reaching the LAN, Vault and every app.
|
||||||
|
# Anything that needs a network gets a namespace and a policy of its own.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default-deny
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
@@ -7,6 +7,7 @@ resources:
|
|||||||
- external-secret.yaml
|
- external-secret.yaml
|
||||||
- rbac.yaml
|
- rbac.yaml
|
||||||
- cronjob.yaml
|
- cronjob.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
|
|
||||||
configMapGenerator:
|
configMapGenerator:
|
||||||
- name: dns-sync-script
|
- name: dns-sync-script
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# The only pod here is the dns-sync Job (a PostSync hook): it installs its
|
||||||
|
# Python packages from PyPI and pushes domains/ to TransIP's API, so it
|
||||||
|
# reaches DNS and the internet (not the LAN, the API server or other
|
||||||
|
# namespaces). Nothing connects to it.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
ports:
|
||||||
|
- port: 443
|
||||||
+57
-30
@@ -10,11 +10,11 @@ One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
|||||||
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
||||||
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
35 Argo CD apps (with metrics-server), all Synced/Healthy; every Longhorn
|
||||||
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
|
volume has one replica. authentik 2026.8.5, Vault 2.2.1. Backups go to the
|
||||||
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to MinIO at
|
NAS (192.168.1.99): Longhorn to NFS at 00:00 (7 kept), Velero to Garage
|
||||||
01:00 (daily kept 30 days, weekly 90 days). Gatus on s1 (node001, Hetzner)
|
(MinIO until 2026-10-26) at 01:00 (daily kept 30 days, weekly 90 days).
|
||||||
and s2 (home) watch each other and alert through ntfy; s2 also gets an
|
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
||||||
hourly report from inside the cluster on the backups, Vault's seal, the
|
through ntfy; s2 also gets an hourly report from inside the cluster on the
|
||||||
Argo CD apps and the CNPG clusters.
|
backups, Vault's seal, the Argo CD apps and the CNPG clusters.
|
||||||
Every app namespace has NetworkPolicies, and CI renders and validates all
|
Every app namespace has NetworkPolicies, and CI renders and validates all
|
||||||
apps on every PR. node001 runs mail and tunnels the public hosts home.
|
apps on every PR. node001 runs mail and tunnels the public hosts home.
|
||||||
|
|
||||||
@@ -63,24 +63,25 @@ belong to the group.
|
|||||||
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
|
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
|
||||||
friend.
|
friend.
|
||||||
|
|
||||||
- [ ] Ours to his bucket: Hyper Backup on the Synology, destination "S3
|
- [ ] Ours to his bucket: `rclone sync` from Garage's `velero-backups`
|
||||||
Storage" with his endpoint as a custom server. Client-side encryption
|
(~190 GB) to his bucket through an rclone `crypt` remote, in Docker on
|
||||||
on: the kopia repositories use Velero's default password
|
the NAS like the copy from MinIO (see 5). Encrypted on our side: the
|
||||||
(`static-passw0rd`), so without it he could read everything, the
|
kopia repositories use Velero's default password (`static-passw0rd`),
|
||||||
group's photos included. Keep the encryption key next to the Vault
|
so without it he could read everything, the group's photos included.
|
||||||
unseal key. Source: the shared folder with MinIO's `velero-backups`
|
Keep the crypt passwords next to the Vault unseal key. Daily after the
|
||||||
bucket (~150-200 GB). Daily after the Velero backup, Smart Recycle,
|
Velero backup, ntfy on failure. Hyper Backup, the earlier plan, backs up
|
||||||
mail on failure
|
shared folders; Garage keeps its objects as blocks plus a metadata
|
||||||
|
database, and a copy of that folder taken while Garage runs is not
|
||||||
|
consistent
|
||||||
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
|
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
|
||||||
now, 500 GB to grow), and versioning or object lock on the bucket, so
|
now, 500 GB to grow), and versioning or object lock on the bucket, so
|
||||||
whoever takes over the NAS can't delete the off-site copy as well
|
whoever takes over the NAS can't delete the off-site copy as well
|
||||||
- [ ] His backups with us: Garage (not MinIO, see 5) in Container Manager
|
- [ ] His backups with us: a bucket and key for him, with a quota, in the
|
||||||
on the Synology, in its own shared folder, with a bucket and key for him
|
Garage on the NAS (`nas/garage`, see 5). Only the S3 API public, as
|
||||||
and a quota. Only the S3 API public, as `s3.rubenhensen.nl` through
|
`s3.rubenhensen.nl` through node001 and Envoy with a cert-manager
|
||||||
node001 and Envoy with a cert-manager certificate; the admin API stays
|
certificate; the admin API stays on the LAN. The cluster side (DNS,
|
||||||
on the LAN. The cluster side (DNS, node001 tunnel, Certificate,
|
node001 tunnel, Certificate, HTTPRoute) goes in git. Needed: how much
|
||||||
HTTPRoute) goes in git. Needed: how much space he wants, free space on
|
space he wants, free space on the NAS
|
||||||
the NAS
|
|
||||||
- [ ] Decide which copy of the immich library goes off-site (see 5)
|
- [ ] Decide which copy of the immich library goes off-site (see 5)
|
||||||
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
||||||
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
|
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth.
|
||||||
@@ -108,9 +109,12 @@ friend.
|
|||||||
MinIO runs region `minio`; the script now reads endpoint, region and
|
MinIO runs region `minio`; the script now reads endpoint, region and
|
||||||
bucket from Velero's location. Copying freshrss's kopia repository took
|
bucket from Velero's location. Copying freshrss's kopia repository took
|
||||||
14 of the ~17 minutes
|
14 of the ~17 minutes
|
||||||
- [ ] Run it on a schedule. It needs Docker and the MinIO key, so not in
|
- [x] Ran it again on 2026-10-26 against the copy in Garage
|
||||||
the cluster itself: the Mac (launchd) or node001, ideally with a
|
(daily-backup-20261026010028, same namespaces), before Velero switched
|
||||||
read-only MinIO user
|
over (see 5): passed, in 5 minutes instead of 17
|
||||||
|
- [ ] Run it on a schedule. It needs Docker and an S3 key, so not in the
|
||||||
|
cluster itself: the Mac (launchd) or node001, ideally with a read-only
|
||||||
|
Garage key (`garage key create`, `garage bucket allow --read`)
|
||||||
|
|
||||||
### 3. authentik 2026.2.1 → 2026.8.5
|
### 3. authentik 2026.2.1 → 2026.8.5
|
||||||
|
|
||||||
@@ -193,7 +197,7 @@ Findings from 2026-10-25:
|
|||||||
- Velero copies the CNPG and MariaDB data directories file by file while
|
- Velero copies the CNPG and MariaDB data directories file by file while
|
||||||
the databases run, which is not a consistent copy. The Longhorn backups
|
the databases run, which is not a consistent copy. The Longhorn backups
|
||||||
start from a snapshot and are crash-consistent. CNPG can back up to
|
start from a snapshot and are crash-consistent. CNPG can back up to
|
||||||
MinIO itself (barman cloud plugin), with point-in-time recovery.
|
Garage itself (barman cloud plugin), with point-in-time recovery.
|
||||||
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
|
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
|
||||||
maintenance job pods) was fixed that afternoon by
|
maintenance job pods) was fixed that afternoon by
|
||||||
`velero/skip-emptydir-policy.yaml`.
|
`velero/skip-emptydir-policy.yaml`.
|
||||||
@@ -201,10 +205,31 @@ Findings from 2026-10-25:
|
|||||||
- [ ] Whether the immich library stays in Velero. It stays for now
|
- [ ] Whether the immich library stays in Velero. It stays for now
|
||||||
(2026-10-25); decide again with the off-site copy (2)
|
(2026-10-25); decide again with the off-site copy (2)
|
||||||
- [ ] CNPG backups through the barman cloud plugin
|
- [ ] CNPG backups through the barman cloud plugin
|
||||||
- [ ] Velero's target, MinIO on the NAS, is no longer maintained: no
|
- [x] Velero's target, MinIO on the NAS, is no longer maintained: no
|
||||||
community images since October 2025, maintenance mode since December
|
community images since October 2025, maintenance mode since December
|
||||||
2025, and the repository is archived. Move Velero to the Garage from 2
|
2025, and the repository is archived. Since 2026-10-26 Velero backs up
|
||||||
and switch MinIO off
|
to Garage on the same NAS (`nas/garage`, PR #35). The bucket was copied
|
||||||
|
with rclone (185 GiB in about 6 hours, then a catch-up sync just before
|
||||||
|
the switch), so the existing backups and kopia repositories carried on:
|
||||||
|
after the switch all 55 backups were still listed, and a manual backup
|
||||||
|
of ntfy found the existing repository and used the night's snapshot as
|
||||||
|
its parent. Two things made the NAS slow during the copy: Garage on
|
||||||
|
LMDB with `metadata_fsync`, ~270 disk writes/s while idle (now SQLite,
|
||||||
|
see `nas/garage/garage.toml`), and MinIO's scanner, which kept one disk
|
||||||
|
at 97% busy. The scanner is now at `slowest` (`mc admin config set local
|
||||||
|
scanner speed=slowest` in the MinIO container); it may have slowed
|
||||||
|
Velero's backups to MinIO as well
|
||||||
|
- [ ] Switch MinIO off once Velero has run on Garage for a week (from
|
||||||
|
2026-11-02): stop the container, then delete `/volume1/minio` (343 GiB)
|
||||||
|
and the hand-made Secret `velero/cloud-credentials` (MinIO's key). Only
|
||||||
|
MinIO still has the kopia repositories of namespaces that are gone
|
||||||
|
(~156 GB: nextcloud, ttrss, whisper-priv, whisper-webui, blog, ghost,
|
||||||
|
mbg-wordpress, sogo, sogo-mail, outrun-game, owntracks); decide first
|
||||||
|
whether any of them is worth keeping
|
||||||
|
- [ ] Garage's SQLite WAL (`meta/db.sqlite-wal`) was 1.7 GB on
|
||||||
|
2026-10-26 next to a 270 MB database, and going by the folders in
|
||||||
|
`meta/snapshots`, each 6-hourly metadata snapshot takes about 70
|
||||||
|
minutes. Not growing; check again after a week of normal use
|
||||||
|
|
||||||
### 6. Monitoring
|
### 6. Monitoring
|
||||||
|
|
||||||
@@ -323,8 +348,10 @@ Counted 2026-10-25 over the 73 running containers:
|
|||||||
|
|
||||||
### 8. Renovate and CI
|
### 8. Renovate and CI
|
||||||
|
|
||||||
10 Renovate PRs are open, the oldest from February, and nothing checks a
|
On 2026-10-25 10 Renovate PRs were open, the oldest from February, and
|
||||||
PR before Argo CD applies it.
|
nothing checked a PR before Argo CD applied it. All of them have since been
|
||||||
|
merged or closed (below), none are open (checked 2026-10-25), and CI checks
|
||||||
|
every PR.
|
||||||
|
|
||||||
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
|
- [x] `.github/workflows/validate.yaml` runs `tools/ci/validate.py` on
|
||||||
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
|
every PR and push (2026-10-25): it renders all 36 apps the way Argo CD
|
||||||
@@ -446,7 +473,7 @@ critical / high:
|
|||||||
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
| tensorchord/cloudnative-vectorchord:17.11-1.1.1 | 17 | 136 | immich's database; look for a newer tag |
|
||||||
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
| longhornio/longhorn-instance-manager:v1.7.3 | 11 | 159 | still serves 16 volumes, see below |
|
||||||
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
| actualbudget/actual-server:26.6.0 | 8 | 83 | chart 1.8.9; Renovate now covers it |
|
||||||
| gitea/gitea:1.27.3 | 8 | 43 | PR #26 (28.2.0) |
|
| gitea/gitea:1.27.3 | 8 | 43 | now 28.2.0 (PR #26, 2026-10-25); not scanned again |
|
||||||
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
|
| immich-server / machine-learning v3.3.5 | 7, 7 | 113, 78 | a newer immich release |
|
||||||
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
|
| towfiqi/serpbear:3.1.0 | 6 | 77 | a newer release |
|
||||||
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
|
| rubenhensen/bolletjestrui:main-10268f1 | 5 | 76 | rebuild on a current base image |
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# External Secrets' three pods reach each other, DNS and the API server
|
||||||
|
# (the node, port 16443); the controller also reads Vault
|
||||||
|
# (vault-active.vault.svc:8200, which vault/networkpolicy.yaml lets it
|
||||||
|
# into). Besides their own namespace they take connections only from the
|
||||||
|
# node: the API server calls the webhook from there and the kubelet probes
|
||||||
|
# come from there too (10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). The Password generator behind
|
||||||
|
# gatus/backup-check-token needs no network. The chart's Application
|
||||||
|
# (apps/templates/external-secrets-helm.yaml) only installs the chart, so
|
||||||
|
# this folder gets an Application of its own, external-secrets-folder.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: controller-to-vault
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: external-secrets
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: vault
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: vault
|
||||||
|
ports:
|
||||||
|
- port: 8200
|
||||||
@@ -3,3 +3,4 @@ kind: Kustomization
|
|||||||
resources:
|
resources:
|
||||||
- operator-crds-helm.yaml
|
- operator-crds-helm.yaml
|
||||||
- operator-helm.yaml
|
- operator-helm.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
# The mariadb-operator's three pods reach each other, DNS and the API
|
||||||
|
# server (the node, port 16443); the operator also connects to MariaDB on
|
||||||
|
# 3306 for the databases, users and grants it manages (mbgwp's
|
||||||
|
# mariadb-from-operator lets it in). Besides their own namespace they take
|
||||||
|
# connections only from the node: the API server calls the webhook from
|
||||||
|
# there and the kubelet probes come from there too (10.1.7.64 is the node's
|
||||||
|
# Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr). This
|
||||||
|
# folder's Application deploys into argocd (the operator's own
|
||||||
|
# Applications), hence the namespace here.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
namespace: mariadb-system
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: mariadb
|
||||||
|
ports:
|
||||||
|
- port: 3306
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: nas-minio-external
|
|
||||||
spec:
|
|
||||||
type: ExternalName
|
|
||||||
externalName: 192.168.1.99
|
|
||||||
ports:
|
|
||||||
- port: 9000
|
|
||||||
protocol: TCP
|
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Velero, its node agent, and the pods it starts here (the data movers of
|
||||||
|
# each pod volume backup or restore, kopia's maintenance jobs, the chart's
|
||||||
|
# CRD upgrade job) reach each other, DNS, the API server (the node, port
|
||||||
|
# 16443) and Garage on the NAS (192.168.1.99:3900, nas/garage). The node
|
||||||
|
# agent reads volumes through its hostPath mount, not the network. Besides
|
||||||
|
# their own namespace they take connections only from the node (kubelet
|
||||||
|
# probes; 10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr); nothing scrapes the metrics on
|
||||||
|
# 8085. MinIO (the NAS, port 9000) is not allowed: going back to it means
|
||||||
|
# adding its port here as well.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.99/32
|
||||||
|
ports:
|
||||||
|
- port: 3900
|
||||||
@@ -27,20 +27,22 @@ spec:
|
|||||||
- mountPath: /target
|
- mountPath: /target
|
||||||
name: plugins
|
name: plugins
|
||||||
configuration:
|
configuration:
|
||||||
|
# Garage on the NAS (nas/garage). Until 2026-10-26 this was MinIO on
|
||||||
|
# the same NAS; the bucket was copied over with rclone, so the
|
||||||
|
# existing backups and kopia repositories carried on.
|
||||||
backupStorageLocation:
|
backupStorageLocation:
|
||||||
- name: default
|
- name: default
|
||||||
bucket: velero-backups
|
bucket: velero-backups
|
||||||
provider: aws
|
provider: aws
|
||||||
config:
|
config:
|
||||||
region: minio
|
region: garage
|
||||||
s3ForcePathStyle: true
|
s3ForcePathStyle: true
|
||||||
s3Url: http://192.168.1.99:9000
|
s3Url: http://192.168.1.99:3900
|
||||||
insecureSkipTLSVerify: "true"
|
|
||||||
volumeSnapshotLocation:
|
volumeSnapshotLocation:
|
||||||
- name: default
|
- name: default
|
||||||
provider: aws
|
provider: aws
|
||||||
config:
|
config:
|
||||||
region: minio
|
region: garage
|
||||||
features: EnableCSI
|
features: EnableCSI
|
||||||
# Kopia maintenance ran every hour for each of the 15 repositories,
|
# Kopia maintenance ran every hour for each of the 15 repositories,
|
||||||
# and once a day as a full run of several minutes that writes its
|
# and once a day as a full run of several minutes that writes its
|
||||||
@@ -59,11 +61,8 @@ spec:
|
|||||||
memoryRequest: 256Mi
|
memoryRequest: 256Mi
|
||||||
memoryLimit: 2Gi
|
memoryLimit: 2Gi
|
||||||
credentials:
|
credentials:
|
||||||
# Use existing secret created by External Secrets Operator
|
# From Vault, velero/external-secret.yaml
|
||||||
# existingSecret: velero-minio-credentials
|
existingSecret: velero-s3-credentials
|
||||||
# extraEnvVars:
|
|
||||||
# AWS_SHARED_CREDENTIALS_FILE: /credentials/cloud
|
|
||||||
existingSecret: cloud-credentials
|
|
||||||
deployNodeAgent: true
|
deployNodeAgent: true
|
||||||
nodeAgent:
|
nodeAgent:
|
||||||
containerSecurityContext:
|
containerSecurityContext:
|
||||||
|
|||||||
Reference in new issue
Block a user