mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-17 14:29:09 +01:00
Compare commits
28
Commits
0ff174d8cb
...
6206ab5a74
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6206ab5a74 | ||
|
|
67c21c30d8 | ||
|
|
0c52d8a467 | ||
|
|
f2e4c5aff6 | ||
|
|
d836e4608c | ||
|
|
1f56e67e8b | ||
|
|
cadac4cebf | ||
|
|
bc71061db7 | ||
|
|
7d3a64e5c8 | ||
|
|
e137e79b75 | ||
|
|
67dc4f0783 | ||
|
|
684f536f2f | ||
|
|
1439466af4 | ||
|
|
57514c3c14 | ||
|
|
57851bf43a | ||
|
|
ada909ee02 | ||
|
|
345de16552 | ||
|
|
7396cc3ed6 | ||
|
|
26ca83457c | ||
|
|
4709194483 | ||
|
|
ad2d6fbae5 | ||
|
|
077711472e | ||
|
|
43f741141d | ||
|
|
cbee4955d3 | ||
|
|
a70dff6fdd | ||
|
|
ab0a09a356 | ||
|
|
b2649a682c | ||
|
|
ce3846bcd3 |
No files matched your search
@@ -0,0 +1,56 @@
|
|||||||
|
# Renders every Argo CD app the way Argo CD does and validates the result
|
||||||
|
# (tools/ci/validate.py), so a broken manifest or a Renovate bump that no
|
||||||
|
# longer renders shows up on the PR instead of in a failed sync.
|
||||||
|
name: validate
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: validate-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
validate:
|
||||||
|
runs-on: ubuntu-24.04
|
||||||
|
timeout-minutes: 20
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Install helm, kustomize and kubeconform
|
||||||
|
env:
|
||||||
|
# Helm 4.2, like Argo CD 3.5.
|
||||||
|
# renovate: datasource=github-releases depName=helm/helm
|
||||||
|
HELM_VERSION: v4.2.1
|
||||||
|
# renovate: datasource=github-releases depName=kubernetes-sigs/kustomize extractVersion=^kustomize/(?<version>.+)$
|
||||||
|
KUSTOMIZE_VERSION: v5.8.3
|
||||||
|
# renovate: datasource=github-releases depName=yannh/kubeconform
|
||||||
|
KUBECONFORM_VERSION: v0.8.0
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$RUNNER_TEMP"
|
||||||
|
curl -sSfLO "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz"
|
||||||
|
curl -sSfL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz.sha256sum" | sha256sum -c -
|
||||||
|
k="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}"
|
||||||
|
curl -sSfLO "$k/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz"
|
||||||
|
curl -sSfL "$k/checksums.txt" | grep " kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz$" | sha256sum -c -
|
||||||
|
c="https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}"
|
||||||
|
curl -sSfLO "$c/kubeconform-linux-amd64.tar.gz"
|
||||||
|
curl -sSfL "$c/CHECKSUMS" | grep " kubeconform-linux-amd64.tar.gz$" | sha256sum -c -
|
||||||
|
mkdir -p bin
|
||||||
|
tar xzf "helm-${HELM_VERSION}-linux-amd64.tar.gz" -C bin --strip-components=1 linux-amd64/helm
|
||||||
|
tar xzf "kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" -C bin kustomize
|
||||||
|
tar xzf kubeconform-linux-amd64.tar.gz -C bin kubeconform
|
||||||
|
echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
|
||||||
|
pip install --quiet pyyaml==6.0.3
|
||||||
|
|
||||||
|
- name: Render and validate
|
||||||
|
run: python3 tools/ci/validate.py
|
||||||
@@ -6,3 +6,4 @@ velero-credentials
|
|||||||
nix-infra-machine/ssh/
|
nix-infra-machine/ssh/
|
||||||
nix-infra-machine/.env
|
nix-infra-machine/.env
|
||||||
nix-infra-machine/nix/
|
nix-infra-machine/nix/
|
||||||
|
.DS_Store
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# Actual faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces.
|
||||||
|
# The OIDC login goes through authentik's public address.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 5006
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
sources:
|
sources:
|
||||||
- chart: authentik
|
- chart: authentik
|
||||||
repoURL: https://charts.goauthentik.io
|
repoURL: https://charts.goauthentik.io
|
||||||
targetRevision: 2026.2.1
|
targetRevision: 2026.8.5
|
||||||
helm:
|
helm:
|
||||||
values: |
|
values: |
|
||||||
authentik:
|
authentik:
|
||||||
@@ -270,12 +270,6 @@ spec:
|
|||||||
readOnly: true
|
readOnly: true
|
||||||
postgresql:
|
postgresql:
|
||||||
enabled: false
|
enabled: false
|
||||||
redis:
|
|
||||||
enabled: true
|
|
||||||
master:
|
|
||||||
persistence:
|
|
||||||
storageClass: longhorn
|
|
||||||
size: 2Gi
|
|
||||||
- repoURL: https://github.com/rubenhensen/k8scd.git
|
- repoURL: https://github.com/rubenhensen/k8scd.git
|
||||||
targetRevision: HEAD
|
targetRevision: HEAD
|
||||||
path: authentik
|
path: authentik
|
||||||
|
|||||||
@@ -1,21 +0,0 @@
|
|||||||
apiVersion: argoproj.io/v1alpha1
|
|
||||||
kind: Application
|
|
||||||
metadata:
|
|
||||||
name: secrets-folder
|
|
||||||
namespace: argocd
|
|
||||||
finalizers:
|
|
||||||
- resources-finalizer.argocd.argoproj.io
|
|
||||||
spec:
|
|
||||||
project: default
|
|
||||||
source:
|
|
||||||
repoURL: https://github.com/rubenhensen/k8scd.git
|
|
||||||
targetRevision: HEAD
|
|
||||||
path: secrets
|
|
||||||
destination:
|
|
||||||
server: https://kubernetes.default.svc
|
|
||||||
namespace: secrets
|
|
||||||
syncPolicy:
|
|
||||||
syncOptions:
|
|
||||||
- CreateNamespace=true
|
|
||||||
automated:
|
|
||||||
selfHeal: true
|
|
||||||
@@ -30,10 +30,8 @@ spec:
|
|||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: kube-system
|
namespace: kube-system
|
||||||
syncPolicy:
|
syncPolicy:
|
||||||
|
# Was listed twice, with and without prune; Argo CD kept the last one.
|
||||||
automated:
|
automated:
|
||||||
selfHeal: true
|
selfHeal: true
|
||||||
prune: true
|
|
||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
automated:
|
|
||||||
selfHeal: true
|
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
# authentik faces the internet (the login for every member of the group),
|
||||||
|
# so its pods only reach each other, DNS and the internet: not the LAN
|
||||||
|
# (NAS, router) or other namespaces. Scouts Online, the apps' OIDC calls
|
||||||
|
# and the LDAP outpost's connection to the server all use public
|
||||||
|
# addresses. The worker and the CNPG instance manager also reach the API
|
||||||
|
# server (the node, port 16443). Ingress comes from Envoy, the CNPG
|
||||||
|
# operator, the node itself (kubelet probes; 10.1.7.64 is the node's
|
||||||
|
# Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr), and for
|
||||||
|
# the LDAP outpost from anywhere: Stalwart on node001 comes in through the
|
||||||
|
# router and the ldap-outpost-external NodePort.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: server-from-envoy
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: authentik
|
||||||
|
app.kubernetes.io/component: server
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 9000
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: ldap-outpost
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: authentik-outpost-ldap
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- ports:
|
||||||
|
- port: 3389
|
||||||
|
- port: 6636
|
||||||
|
---
|
||||||
|
# The outpost controller deploys and updates the LDAP outpost.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: worker-to-api
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: authentik
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
---
|
||||||
|
# The CNPG operator checks each instance on its status port and connects to
|
||||||
|
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
||||||
|
# through the API server.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: postgres-cnpg
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: cnpg.io/cluster
|
||||||
|
operator: Exists
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: cnpg-system
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
- port: 5432
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
@@ -1,381 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# test-velero-backup.sh - Automated script to test Velero backups
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# Configuration - customize these variables
|
|
||||||
BACKUP_NAME="" # Will list backups if empty
|
|
||||||
S3_BUCKET="velero-backups"
|
|
||||||
S3_REGION="minio"
|
|
||||||
S3_URL="http://192.168.1.99:9000" # Your NAS MinIO server
|
|
||||||
S3_ACCESS_KEY="" # Will be filled from credentials
|
|
||||||
S3_SECRET_KEY="" # Will be filled from credentials
|
|
||||||
ORIGINAL_NAMESPACE="" # Namespace in the original backup
|
|
||||||
TEST_NAMESPACE="default"
|
|
||||||
DEBUG=false # Set to true for verbose debugging
|
|
||||||
|
|
||||||
# Parse command line arguments
|
|
||||||
while [[ $# -gt 0 ]]; do
|
|
||||||
case $1 in
|
|
||||||
--backup-name)
|
|
||||||
BACKUP_NAME="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--s3-access-key)
|
|
||||||
S3_ACCESS_KEY="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--s3-secret-key)
|
|
||||||
S3_SECRET_KEY="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--s3-bucket)
|
|
||||||
S3_BUCKET="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--s3-region)
|
|
||||||
S3_REGION="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--s3-url)
|
|
||||||
S3_URL="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--original-namespace)
|
|
||||||
ORIGINAL_NAMESPACE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--test-namespace)
|
|
||||||
TEST_NAMESPACE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--debug)
|
|
||||||
DEBUG=true
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--help)
|
|
||||||
echo "Usage: $0 [options]"
|
|
||||||
echo "Options:"
|
|
||||||
echo " --backup-name NAME Name of the Velero backup to restore"
|
|
||||||
echo " --s3-bucket BUCKET S3 bucket containing backups"
|
|
||||||
echo " --s3-region REGION S3 region"
|
|
||||||
echo " --s3-url URL S3 endpoint URL"
|
|
||||||
echo " --s3-access-key KEY S3 access key"
|
|
||||||
echo " --s3-secret-key KEY S3 secret key"
|
|
||||||
echo " --original-namespace NS Namespace in the original backup"
|
|
||||||
echo " --test-namespace NS Namespace for restoration (default: default)"
|
|
||||||
echo " --debug Enable debug output"
|
|
||||||
echo " --help Show this help message"
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown option: $1"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
# Verify required variables
|
|
||||||
if [[ -z "$S3_BUCKET" || -z "$S3_REGION" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" || -z "$ORIGINAL_NAMESPACE" || -z "$TEST_NAMESPACE" ]]; then
|
|
||||||
echo "Error: S3 configuration is incomplete"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Function to clean up resources
|
|
||||||
cleanup() {
|
|
||||||
echo "Cleaning up resources..."
|
|
||||||
k3d cluster delete backup-test || true
|
|
||||||
echo "Cleanup complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Function for debugging output
|
|
||||||
debug() {
|
|
||||||
if [[ "$DEBUG" == "true" ]]; then
|
|
||||||
echo "[DEBUG] $*"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Register cleanup function to run on exit (ctrl+c, etc.)
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
echo "=== Starting Velero backup test process ==="
|
|
||||||
|
|
||||||
# 1. Create K3d cluster
|
|
||||||
echo "Creating K3d cluster..."
|
|
||||||
k3d cluster create backup-test \
|
|
||||||
--api-port 6443 \
|
|
||||||
--port "80:80@loadbalancer" \
|
|
||||||
--port "443:443@loadbalancer" \
|
|
||||||
--agents 0 \
|
|
||||||
--k3s-arg "--disable=traefik@server:0" \
|
|
||||||
--wait
|
|
||||||
# -v /mnt/longhorn:/var/lib/longhorn:shared@all \
|
|
||||||
# -i hebury/k3s:v1.32.2-k3s1 \
|
|
||||||
# --wait
|
|
||||||
|
|
||||||
# Verify cluster is running
|
|
||||||
kubectl config use-context k3d-backup-test
|
|
||||||
kubectl get nodes
|
|
||||||
|
|
||||||
# 2. Install Longhorn
|
|
||||||
# echo "Installing Longhorn..."
|
|
||||||
# kubectl create namespace longhorn-system
|
|
||||||
|
|
||||||
# helm repo add longhorn https://charts.longhorn.io
|
|
||||||
# helm repo update
|
|
||||||
|
|
||||||
# helm install longhorn longhorn/longhorn \
|
|
||||||
# --namespace longhorn-system \
|
|
||||||
# --set persistence.defaultClassReplicaCount=1 \
|
|
||||||
# --set defaultSettings.backupTarget="" \
|
|
||||||
# --set defaultSettings.defaultReplicaCount=1
|
|
||||||
|
|
||||||
# echo "Waiting for Longhorn to be ready..."
|
|
||||||
# kubectl -n longhorn-system rollout status deployment/longhorn-ui
|
|
||||||
# kubectl -n longhorn-system rollout status deployment/longhorn-driver-deployer
|
|
||||||
# kubectl -n longhorn-system rollout status daemonset/longhorn-manager
|
|
||||||
|
|
||||||
# 3. Install Velero with S3 provider
|
|
||||||
echo "Installing Velero..."
|
|
||||||
cat > velero-credentials <<EOF
|
|
||||||
[default]
|
|
||||||
aws_access_key_id=$S3_ACCESS_KEY
|
|
||||||
aws_secret_access_key=$S3_SECRET_KEY
|
|
||||||
EOF
|
|
||||||
|
|
||||||
helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
|
|
||||||
helm repo update
|
|
||||||
|
|
||||||
debug "Installing Velero with S3 configuration:"
|
|
||||||
debug " Bucket: $S3_BUCKET"
|
|
||||||
debug " Region: $S3_REGION"
|
|
||||||
debug " URL: $S3_URL"
|
|
||||||
|
|
||||||
helm install velero vmware-tanzu/velero \
|
|
||||||
--namespace velero \
|
|
||||||
--create-namespace \
|
|
||||||
--set-file credentials.secretContents.cloud=velero-credentials \
|
|
||||||
--set configuration.backupStorageLocation[0].provider=aws \
|
|
||||||
--set configuration.backupStorageLocation[0].name=default \
|
|
||||||
--set configuration.backupStorageLocation[0].bucket=$S3_BUCKET \
|
|
||||||
--set configuration.backupStorageLocation[0].config.region=$S3_REGION \
|
|
||||||
--set configuration.backupStorageLocation[0].config.s3ForcePathStyle=true \
|
|
||||||
--set configuration.backupStorageLocation[0].config.s3Url=$S3_URL \
|
|
||||||
--set configuration.volumeSnapshotLocation[0].name=default \
|
|
||||||
--set configuration.volumeSnapshotLocation[0].provider=aws \
|
|
||||||
--set configuration.volumeSnapshotLocation[0].config.region=$S3_REGION \
|
|
||||||
--set configuration.features=EnableCSI \
|
|
||||||
--set initContainers[0].name=velero-plugin-for-aws \
|
|
||||||
--set initContainers[0].image=velero/velero-plugin-for-aws:v1.11.1 \
|
|
||||||
--set initContainers[0].volumeMounts[0].mountPath=/target \
|
|
||||||
--set initContainers[0].volumeMounts[0].name=plugins \
|
|
||||||
--set deployNodeAgent=true \
|
|
||||||
--set nodeAgent.containerSecurityContext.privileged=true \
|
|
||||||
--wait
|
|
||||||
|
|
||||||
rm velero-credentials
|
|
||||||
|
|
||||||
# Wait for Velero to be ready
|
|
||||||
echo "Waiting for Velero to be fully ready..."
|
|
||||||
kubectl -n velero rollout status deployment/velero --timeout=120s
|
|
||||||
|
|
||||||
echo "Install change-storage-class.yaml"
|
|
||||||
kubectl apply -f change-storage-class.yaml
|
|
||||||
|
|
||||||
# Check if velero CLI is installed and functioning
|
|
||||||
if ! command -v velero &> /dev/null; then
|
|
||||||
echo "Error: Velero CLI is not installed or not in PATH"
|
|
||||||
echo "Please install the Velero CLI by following instructions at: https://velero.io/docs/main/basic-install/"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Verify Velero is connected to the storage location
|
|
||||||
echo "Checking Velero backup storage location..."
|
|
||||||
kubectl -n velero get backupstoragelocation default -o jsonpath='{.status.phase}' | grep -q "Available" || {
|
|
||||||
echo "Warning: Backup storage location is not available, it may take a few minutes to initialize"
|
|
||||||
echo "Storage location status:"
|
|
||||||
kubectl -n velero get backupstoragelocation default -o yaml
|
|
||||||
|
|
||||||
# Wait a bit longer for the location to become available
|
|
||||||
echo "Waiting 30 seconds for backup storage location to become available..."
|
|
||||||
sleep 30
|
|
||||||
}
|
|
||||||
|
|
||||||
# 4. List backups
|
|
||||||
echo "Checking for available backups..."
|
|
||||||
if ! velero_output=$(velero backup get 2>&1); then
|
|
||||||
echo "Error listing backups: $velero_output"
|
|
||||||
|
|
||||||
# Try to verify S3 connectivity
|
|
||||||
echo "Checking S3 connectivity..."
|
|
||||||
|
|
||||||
# Install AWS CLI if needed
|
|
||||||
if ! command -v aws &> /dev/null; then
|
|
||||||
echo "AWS CLI not found, skipping S3 connectivity test"
|
|
||||||
else
|
|
||||||
# Create temporary AWS profile
|
|
||||||
mkdir -p ~/.aws
|
|
||||||
cat > ~/.aws/credentials <<EOF
|
|
||||||
[velero-test]
|
|
||||||
aws_access_key_id=$S3_ACCESS_KEY
|
|
||||||
aws_secret_access_key=$S3_SECRET_KEY
|
|
||||||
EOF
|
|
||||||
|
|
||||||
cat > ~/.aws/config <<EOF
|
|
||||||
[profile velero-test]
|
|
||||||
region=$S3_REGION
|
|
||||||
output=json
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Try to list bucket contents
|
|
||||||
echo "Testing S3 connection to bucket $S3_BUCKET..."
|
|
||||||
if aws --profile velero-test --endpoint-url $S3_URL s3 ls s3://$S3_BUCKET/; then
|
|
||||||
echo "S3 connection successful, but Velero couldn't list backups."
|
|
||||||
echo "This might indicate that there are no backups in the bucket or the backups are in a different format than expected."
|
|
||||||
else
|
|
||||||
echo "Error connecting to S3. Please check your credentials and connectivity to $S3_URL"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Cleanup AWS files
|
|
||||||
rm ~/.aws/credentials ~/.aws/config
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "$BACKUP_NAME" ]]; then
|
|
||||||
echo "No backup name specified and no backups found. Exiting."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Available backups:"
|
|
||||||
echo "$velero_output"
|
|
||||||
|
|
||||||
if [[ -z "$BACKUP_NAME" ]]; then
|
|
||||||
echo "Specify a backup name using --backup-name and run again."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 5. Create test namespace if needed
|
|
||||||
if [[ "$TEST_NAMESPACE" != "default" ]]; then
|
|
||||||
kubectl create namespace $TEST_NAMESPACE || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 6. Restore backup
|
|
||||||
echo "Restoring backup: $BACKUP_NAME"
|
|
||||||
if [[ -z "$ORIGINAL_NAMESPACE" ]]; then
|
|
||||||
# If no namespace specified, restore everything
|
|
||||||
velero restore create --from-backup $BACKUP_NAME \
|
|
||||||
--wait
|
|
||||||
else
|
|
||||||
# Restore specific namespace with mapping
|
|
||||||
velero restore create --from-backup $BACKUP_NAME \
|
|
||||||
--namespace-mappings $ORIGINAL_NAMESPACE:$TEST_NAMESPACE \
|
|
||||||
--include-namespaces $ORIGINAL_NAMESPACE \
|
|
||||||
--exclude-namespaces kube-system,kube-public,kube-node-lease,velero \
|
|
||||||
--exclude-resources certificates.cert-manager.io \
|
|
||||||
--wait
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 7. Check restore status
|
|
||||||
echo "Checking restore status:"
|
|
||||||
velero restore get
|
|
||||||
|
|
||||||
# Wait for pods to be ready
|
|
||||||
echo "Waiting for pods to be ready..."
|
|
||||||
kubectl wait --for=condition=ready pod --all -n $TEST_NAMESPACE --timeout=600s || true
|
|
||||||
|
|
||||||
# 8. Run validation script (create it first if it doesn't exist)
|
|
||||||
# if [[ ! -f ./validate-restore.sh ]]; then
|
|
||||||
# echo "Creating validation script..."
|
|
||||||
# cat > validate-restore.sh <<'EOF'
|
|
||||||
# #!/bin/bash
|
|
||||||
# # validate-restore.sh - Script to validate Velero backup restoration
|
|
||||||
|
|
||||||
# set -e
|
|
||||||
# echo "Starting validation of restored resources..."
|
|
||||||
|
|
||||||
# # Get namespace from command line or use default
|
|
||||||
# NAMESPACE="${1:-default}"
|
|
||||||
|
|
||||||
# # Check all deployments
|
|
||||||
# echo "Checking deployments..."
|
|
||||||
# deployments=$(kubectl get deployment -n $NAMESPACE -o name)
|
|
||||||
# if [[ -z "$deployments" ]]; then
|
|
||||||
# echo "❌ No deployments found in namespace $NAMESPACE"
|
|
||||||
# else
|
|
||||||
# echo "Found deployments: $deployments"
|
|
||||||
|
|
||||||
# # Check if pods are running for each deployment
|
|
||||||
# for deployment in $deployments; do
|
|
||||||
# name=$(echo $deployment | cut -d'/' -f2)
|
|
||||||
# echo -n "Checking deployment $name: "
|
|
||||||
|
|
||||||
# # Check if pods are running
|
|
||||||
# ready_replicas=$(kubectl get deployment -n $NAMESPACE $name -o jsonpath='{.status.readyReplicas}')
|
|
||||||
# if [[ -n "$ready_replicas" && "$ready_replicas" != "0" ]]; then
|
|
||||||
# echo "✅ $ready_replicas pods ready"
|
|
||||||
# else
|
|
||||||
# echo "❌ No pods running"
|
|
||||||
# fi
|
|
||||||
# done
|
|
||||||
# fi
|
|
||||||
|
|
||||||
# # Check all services
|
|
||||||
# echo "Checking services..."
|
|
||||||
# services=$(kubectl get service -n $NAMESPACE -o name | grep -v "kubernetes")
|
|
||||||
# if [[ -z "$services" ]]; then
|
|
||||||
# echo "❌ No services found in namespace $NAMESPACE"
|
|
||||||
# else
|
|
||||||
# echo "Found services: $services"
|
|
||||||
|
|
||||||
# # Check if services have endpoints
|
|
||||||
# for service in $services; do
|
|
||||||
# name=$(echo $service | cut -d'/' -f2)
|
|
||||||
# echo -n "Checking service $name: "
|
|
||||||
|
|
||||||
# # Check if service has endpoints
|
|
||||||
# endpoints=$(kubectl get endpoints -n $NAMESPACE $name -o jsonpath='{.subsets[*].addresses[*].ip}')
|
|
||||||
# if [ -n "$endpoints" ]; then
|
|
||||||
# echo "✅ Has endpoints"
|
|
||||||
# else
|
|
||||||
# echo "❌ No endpoints"
|
|
||||||
# fi
|
|
||||||
# done
|
|
||||||
# fi
|
|
||||||
|
|
||||||
# # Check PVCs
|
|
||||||
# echo "Checking PVCs..."
|
|
||||||
# pvcs=$(kubectl get pvc -n $NAMESPACE -o name)
|
|
||||||
# if [[ -z "$pvcs" ]]; then
|
|
||||||
# echo "No PVCs found in namespace $NAMESPACE"
|
|
||||||
# else
|
|
||||||
# pvc_count=$(echo "$pvcs" | wc -l)
|
|
||||||
# echo "Found $pvc_count PVCs"
|
|
||||||
|
|
||||||
# # Check if PVCs are bound
|
|
||||||
# bound_count=$(kubectl get pvc -n $NAMESPACE -o jsonpath='{.items[?(@.status.phase=="Bound")].metadata.name}' | wc -w)
|
|
||||||
# echo "$bound_count/$pvc_count PVCs are bound"
|
|
||||||
|
|
||||||
# if [ "$bound_count" -ne "$pvc_count" ]; then
|
|
||||||
# echo "❌ Not all PVCs are bound"
|
|
||||||
# else
|
|
||||||
# echo "✅ All PVCs are bound"
|
|
||||||
# fi
|
|
||||||
# fi
|
|
||||||
|
|
||||||
# echo "Validation complete!"
|
|
||||||
# exit 0
|
|
||||||
# EOF
|
|
||||||
# chmod +x validate-restore.sh
|
|
||||||
# fi
|
|
||||||
|
|
||||||
echo "Running validation tests..."
|
|
||||||
./validate-restore.sh $TEST_NAMESPACE
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Backup test process completed ==="
|
|
||||||
echo "The cluster will be deleted when you close this terminal or press Ctrl+C"
|
|
||||||
echo "To access the cluster manually, use: kubectl config use-context k3d-backup-test"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Keep the script running until user terminates it
|
|
||||||
read -p "Press Enter to clean up and exit..."
|
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# Bolletjestrui faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 3000
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -31,8 +31,9 @@ view, and everything is shared with everyone automatically.
|
|||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
- **authentik** (authentik.rubenhensen.nl for now; auth.scoutingmbg.nl
|
- **authentik** (on auth.scoutingmbg.nl, with a brand "Scouting MBG" that
|
||||||
later, once the group's DNS has a record for it) brokers the login: a
|
logs in through Scouts Online and then sends people to the photos)
|
||||||
|
brokers the login: a
|
||||||
Scouts Online source turns the roles into the groups `MBG leden` and
|
Scouts Online source turns the roles into the groups `MBG leden` and
|
||||||
`MBG kader` (active `Kaderlid` role) on every login, and the immich
|
`MBG kader` (active `Kaderlid` role) on every login, and the immich
|
||||||
provider turns those into `immich_quota` and `immich_role`
|
provider turns those into `immich_quota` and `immich_role`
|
||||||
@@ -106,6 +107,20 @@ C. Switch
|
|||||||
- [x] Nobody uses `fotoupload@` anymore: its 9 (browser) sessions ended
|
- [x] Nobody uses `fotoupload@` anymore: its 9 (browser) sessions ended
|
||||||
2026-10-24. Its API key stays for share-sync
|
2026-10-24. Its API key stays for share-sync
|
||||||
|
|
||||||
|
D. auth.scoutingmbg.nl (2026-10-25)
|
||||||
|
- [x] The group's DNS points scoutingmbg.nl, www, foto and auth at node001
|
||||||
|
(rubenhensen.nl). node001 passes all four through to the home cluster
|
||||||
|
(SNI on 443, a proxy on 80 for HTTP-01), like its other tunneled hosts,
|
||||||
|
and gatus on s1 checks them
|
||||||
|
- [x] Certificate, Gateway listener `https-auth-scoutingmbg`, the authentik
|
||||||
|
route on both hostnames, the brand, and immich's issuer on
|
||||||
|
https://auth.scoutingmbg.nl/application/o/immich/
|
||||||
|
- [x] SOL connection: redirect URI
|
||||||
|
`https://auth.scoutingmbg.nl/source/oauth/callback/scouts-online/`.
|
||||||
|
Editing the connection left the authentik.rubenhensen.nl URI unusable
|
||||||
|
(SOL answers `invalid_client` after the login when the redirect URI isn't
|
||||||
|
registered), so logins failed for a few minutes until immich moved over
|
||||||
|
|
||||||
## Open points
|
## Open points
|
||||||
|
|
||||||
- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
|
- DNS (2026-10-24): the group's DNS host (ns1/ns2.webhosted.nl) changed
|
||||||
|
|||||||
@@ -45,6 +45,15 @@ between them live.
|
|||||||
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
longer exists), 56.6 GiB from 2026-01-20. The Longhorn backups on the NAS
|
||||||
don't depend on them. Deleting a snapshot merges it into the next one,
|
don't depend on them. Deleting a snapshot merges it into the next one,
|
||||||
which writes a lot, so do this in a quiet hour, one volume at a time.
|
which writes a lot, so do this in a quiet hour, one volume at a time.
|
||||||
|
Happened on 2026-10-24 (who started it isn't in the logs): immich-claim's
|
||||||
|
old snapshots were removed at 14:43 UTC and Longhorn merged them one
|
||||||
|
after another, about 45 minutes each, until 23:15. For those 8.5 hours
|
||||||
|
sda ran 250-440 writes/s and 200-400 reads/s at 6-20 ms per write on
|
||||||
|
average, with "full" IO pressure at 45-75% (`~/iosample.log`); the
|
||||||
|
instance manager wrote 2.3 TB and read 2.8 TB in 30 hours. What is left
|
||||||
|
on immich-claim is one 233 GiB snapshot (`system-backup-0b7f8c6047294b36`,
|
||||||
|
from the Longhorn system backup of 2026-10-22) that everything merged
|
||||||
|
into; 148 snapshots hold 273 GiB across all volumes.
|
||||||
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
|
- [ ] Trim immich-claim's filesystem (Longhorn UI, volume, "Trim Filesystem")
|
||||||
so blocks of deleted files are released.
|
so blocks of deleted files are released.
|
||||||
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
|
- [ ] Fresh backups: let a nightly Longhorn and Velero backup run, plus a
|
||||||
@@ -61,8 +70,10 @@ between them live.
|
|||||||
- [ ] Boot into the BIOS (F2): the NVMe drive shows up under storage, SATA
|
- [ ] Boot into the BIOS (F2): the NVMe drive shows up under storage, SATA
|
||||||
operation stays AHCI, and the boot sequence still starts "ubuntu" from
|
operation stays AHCI, and the boot sequence still starts "ubuntu" from
|
||||||
the SATA SSD.
|
the SATA SSD.
|
||||||
- [ ] Boot. `lsblk` shows `nvme0n1`; wait until everything is green again
|
- [ ] Boot. `lsblk` shows `nvme0n1`.
|
||||||
(gatus on s1 and s2).
|
- [ ] Unseal Vault, which comes up sealed (until then ESO can't refresh
|
||||||
|
secrets): `for p in vault-0 vault-1 vault-2; do kubectl -n vault exec -ti $p -- vault operator unseal; done`
|
||||||
|
- [ ] Wait until everything is green again (gatus on s1 and s2).
|
||||||
|
|
||||||
## Prepare the disk (sudo on opti1)
|
## Prepare the disk (sudo on opti1)
|
||||||
|
|
||||||
|
|||||||
+307
@@ -0,0 +1,307 @@
|
|||||||
|
# Roadmap
|
||||||
|
|
||||||
|
Where the cluster stands and what is left, as of 2026-10-25. Takes over
|
||||||
|
from wishlist.txt (January 2026, removed); its items are sorted in at the
|
||||||
|
bottom.
|
||||||
|
|
||||||
|
## State
|
||||||
|
|
||||||
|
One node: opti1, an OptiPlex 5060 with an i5-8600T, 32 GB and one 1 TB
|
||||||
|
SATA SSD that is 72% full. microk8s v1.36.5 from 1.36/edge, snap held.
|
||||||
|
35 Argo CD apps, all Synced/Healthy; every Longhorn volume has one
|
||||||
|
replica. Backups go to the NAS (192.168.1.99): Longhorn to NFS at 00:00
|
||||||
|
(7 kept), Velero to MinIO at 01:00 (daily kept 30 days, weekly 90 days).
|
||||||
|
Gatus on s1 (node001, Hetzner) and s2 (home) watch each other and alert
|
||||||
|
through ntfy. node001 runs mail and tunnels the public hosts home.
|
||||||
|
|
||||||
|
## Follow-ups from this week
|
||||||
|
|
||||||
|
From docs/upgrade-k8s-1.33.md, docs/immich-scouts-online.md and
|
||||||
|
docs/longhorn-second-ssd.md:
|
||||||
|
|
||||||
|
- [x] Read `~/iosample.log` on opti1: the worst stretch was not the
|
||||||
|
backups but 8.5 hours of Longhorn merging immich-claim's deleted
|
||||||
|
snapshots (2026-10-24 14:30-23:15), see docs/upgrade-k8s-1.33.md
|
||||||
|
- [ ] From 2026-10-27, if immich runs without problems: delete
|
||||||
|
`immich/postgresql-cluster.yaml` (the PG14 cluster still runs, 20 GiB),
|
||||||
|
the Longhorn snapshots and backups `immich-pg14-pre-vchord` and
|
||||||
|
`immich-pg17-pre-v3`, and both dumps in `~` on opti1. PostgreSQL 14
|
||||||
|
reaches end of life on 2026-11-12
|
||||||
|
- [ ] The upgrade's dqlite and backend copies in `/root/` on opti1
|
||||||
|
(`backend-*.tar`, `dqlite-pre-*`)
|
||||||
|
- [ ] Log in to immich as a jeugdlid (quota 0)
|
||||||
|
- [ ] Back to 1.36/stable once it is at v1.36.5 or later. The snap is held,
|
||||||
|
so microk8s gets no updates at all until someone refreshes it
|
||||||
|
- [ ] Second SSD, docs/longhorn-second-ssd.md
|
||||||
|
- [ ] Optional: `MACHINE_LEARNING_MODEL_REVISION=v2` for immich ML
|
||||||
|
|
||||||
|
## Next, in order
|
||||||
|
|
||||||
|
### 1. hensen.io expires
|
||||||
|
|
||||||
|
foto.scoutingmbg.nl was a CNAME to mbg.hensen.io. Since 2026-10-24 it is a
|
||||||
|
CNAME to rubenhensen.nl (node001, which tunnels it home), but only on the
|
||||||
|
group's ns2: ns1 is stuck on an old serial and still answers
|
||||||
|
mbg.hensen.io (docs/immich-scouts-online.md).
|
||||||
|
|
||||||
|
- [x] Expiry date: 2027-01-09 (whois, 2026-10-25; registrar Key-Systems)
|
||||||
|
- [ ] Have the group's DNS host fix ns1; keep hensen.io until it answers
|
||||||
|
rubenhensen.nl too
|
||||||
|
|
||||||
|
### 2. Off-site backups
|
||||||
|
|
||||||
|
Both backup targets are on the same NAS, in the same house as opti1. A
|
||||||
|
fire or burglary takes the cluster, the backups and 45,703 photos that
|
||||||
|
belong to the group.
|
||||||
|
|
||||||
|
There is no off-site copy (2026-10-25). The plan: swap S3 buckets with a
|
||||||
|
friend.
|
||||||
|
|
||||||
|
- [ ] Ours to his bucket: Hyper Backup on the Synology, destination "S3
|
||||||
|
Storage" with his endpoint as a custom server. Client-side encryption
|
||||||
|
on: the kopia repositories use Velero's default password
|
||||||
|
(`static-passw0rd`), so without it he could read everything, the
|
||||||
|
group's photos included. Keep the encryption key next to the Vault
|
||||||
|
unseal key. Source: the shared folder with MinIO's `velero-backups`
|
||||||
|
bucket (~150-200 GB). Daily after the Velero backup, Smart Recycle,
|
||||||
|
mail on failure
|
||||||
|
- [ ] Needed from him: endpoint, which S3 server he runs, space (~200 GB
|
||||||
|
now, 500 GB to grow), and versioning or object lock on the bucket, so
|
||||||
|
whoever takes over the NAS can't delete the off-site copy as well
|
||||||
|
- [ ] His backups with us: Garage (not MinIO, see 5) in Container Manager
|
||||||
|
on the Synology, in its own shared folder, with a bucket and key for him
|
||||||
|
and a quota. Only the S3 API public, as `s3.rubenhensen.nl` through
|
||||||
|
node001 and Envoy with a cert-manager certificate; the admin API stays
|
||||||
|
on the LAN. The cluster side (DNS, node001 tunnel, Certificate,
|
||||||
|
HTTPRoute) goes in git. Needed: how much space he wants, free space on
|
||||||
|
the NAS
|
||||||
|
- [ ] Decide which copy of the immich library goes off-site (see 5)
|
||||||
|
- [ ] A restore test that runs on a schedule. `backup-testing/` predates
|
||||||
|
Envoy Gateway and still rewrites `*.hensen.io` and bypasses sol-auth
|
||||||
|
|
||||||
|
### 3. authentik 2026.2.1 → 2026.8.5
|
||||||
|
|
||||||
|
Since 2026-10-24 authentik is the front door for every member of the
|
||||||
|
group. Renovate PRs #20 (2026.2.3) and #22 (2026.8.5) had been open since
|
||||||
|
April and May.
|
||||||
|
|
||||||
|
- [x] Upgraded 2026-10-25 in two hops, since releases can't be skipped:
|
||||||
|
chart 2026.5.6, then 2026.8.5. Each took about three minutes with no
|
||||||
|
downtime; the old pods kept serving until the new ones were ready.
|
||||||
|
Safety net: `pg_dump -Fc` in `~/authentik-pre-2026.5.dump` (15 MB) on
|
||||||
|
opti1 and Longhorn snapshot `authentik-pg-pre-2026-5`. Checked after
|
||||||
|
each hop: the seven OIDC discovery endpoints and their issuers, the
|
||||||
|
login page, immich, every blueprint successful, the LDAP outpost
|
||||||
|
checking in on the new version
|
||||||
|
- 2026.5 listens on `[::]` instead of `0.0.0.0`. IPv6 is on in the pods
|
||||||
|
(`bindv6only` 0), so IPv4 keeps working
|
||||||
|
- 2026.8 only believes `X-Forwarded-*` from
|
||||||
|
`listen.trusted_proxy_cidrs`, which defaults to the private ranges
|
||||||
|
(10.0.0.0/8 covers Envoy's pods); redirects stay https
|
||||||
|
- The managed LDAP outpost didn't follow on its own after either hop;
|
||||||
|
`outpost_controller.send(<outpost pk>)` in `ak shell` rolled it over
|
||||||
|
- The `redis:` values were left over from before authentik dropped
|
||||||
|
Redis; the chart has no such key, so they went
|
||||||
|
- [x] Log in once through each app: immich through Scouts Online, Argo CD,
|
||||||
|
Vault, Longhorn, FreshRSS, Actual, mail
|
||||||
|
- [ ] After a week without problems: delete the dump and the snapshot
|
||||||
|
- [ ] Base URL (system setting or `AUTHENTIK_WEB__BASE_URL`) becomes
|
||||||
|
required in 2026.11. authentik answers on authentik.rubenhensen.nl and
|
||||||
|
auth.scoutingmbg.nl (immich's issuer), so find out first what the
|
||||||
|
setting does with two hostnames
|
||||||
|
|
||||||
|
### 4. Vault 1.17.2 → 2.2.1
|
||||||
|
|
||||||
|
Vault unseals with Shamir keys, so after every reboot (the SSD install,
|
||||||
|
for one) it stays sealed and ESO stops refreshing secrets until someone
|
||||||
|
unseals it.
|
||||||
|
|
||||||
|
- [x] Upgraded 2026-10-25, straight from 1.17.2 to 2.2.1. HashiCorp
|
||||||
|
doesn't require going through every release, and the change tracker
|
||||||
|
lists no breaking change that applies here: `disable_mlock` is set
|
||||||
|
(required for raft since 1.20), the chart sets `SKIP_SETCAP` and runs
|
||||||
|
Vault as uid 100 (2.0.1's IPC_LOCK requirement, dropped again in 2.0.2),
|
||||||
|
and ESO uses a token and KV v2. `tools/vault-roll.sh 2.2.1` restarted
|
||||||
|
the standbys first and the active node last, unsealing each; vault-1
|
||||||
|
took over as active within seconds. Safety net: Longhorn snapshots
|
||||||
|
`vault-{0,1,2}-pre-2-2-1`. Afterwards all three ran 2.2.1 at the same
|
||||||
|
raft index, and all 24 ExternalSecrets refreshed after a force-sync
|
||||||
|
- The OIDC job called `vault auth tune -listing_visibility`, a flag that
|
||||||
|
doesn't exist, so the OIDC mount's listing visibility was never set.
|
||||||
|
Fixed; the job now uses the 2.2.1 CLI instead of 1.15
|
||||||
|
- [ ] There is one unseal key (1 share, threshold 1). Keep it in two
|
||||||
|
places outside this cluster; without it the data in Vault is lost
|
||||||
|
- [x] Unsealing is in the SSD install steps
|
||||||
|
- [ ] After a week without problems: delete the three snapshots
|
||||||
|
|
||||||
|
### 5. Backups: which tool keeps what
|
||||||
|
|
||||||
|
Findings from 2026-10-25:
|
||||||
|
|
||||||
|
- Velero backs up every mounted volume through kopia, including the immich
|
||||||
|
library (133 GB), which Longhorn also backs up to the same NAS. Most runs
|
||||||
|
take 10s to 3 min, since kopia only reads changed files. 3 of the last 13
|
||||||
|
read everything again: 37 min on 10-21, 30 min on 10-24 (the lease loss
|
||||||
|
at 01:09), 13 min for the weekly on 10-25. Why those runs found no
|
||||||
|
usable parent is unclear; the 10-24 one was the first after the Velero
|
||||||
|
1.18 upgrade restarted the node agent. Once Longhorn's data is on the
|
||||||
|
NVMe SSD these reads no longer hit the disk dqlite is on.
|
||||||
|
- Velero holds the photos longest. Longhorn keeps 7 backups per volume
|
||||||
|
(count-based, so extra runs push out older ones: immich-claim's oldest
|
||||||
|
daily is from 10-22); Velero keeps dailies 30 days and weeklies 90.
|
||||||
|
Dropping immich from Velero means raising Longhorn's retention first.
|
||||||
|
- Longhorn's recurring job skips detached volumes
|
||||||
|
(`allow-recurring-job-while-volume-detached` is false). immich-claim got
|
||||||
|
no Longhorn backup on 10-24: the server was at 0 replicas for the
|
||||||
|
database migration from 22:18 to 00:06 UTC.
|
||||||
|
- Velero copies the CNPG and MariaDB data directories file by file while
|
||||||
|
the databases run, which is not a consistent copy. The Longhorn backups
|
||||||
|
start from a snapshot and are crash-consistent. CNPG can back up to
|
||||||
|
MinIO itself (barman cloud plugin), with point-in-time recovery.
|
||||||
|
- The PartiallyFailed daily backup of 10-23 (emptyDirs of Velero's own
|
||||||
|
maintenance job pods) was fixed that afternoon by
|
||||||
|
`velero/skip-emptydir-policy.yaml`.
|
||||||
|
|
||||||
|
- [ ] Whether the immich library stays in Velero. It stays for now
|
||||||
|
(2026-10-25); decide again with the off-site copy (2)
|
||||||
|
- [ ] CNPG backups through the barman cloud plugin
|
||||||
|
- [ ] Velero's target, MinIO on the NAS, is no longer maintained: no
|
||||||
|
community images since October 2025, maintenance mode since December
|
||||||
|
2025, and the repository is archived. Move Velero to the Garage from 2
|
||||||
|
and switch MinIO off
|
||||||
|
|
||||||
|
### 6. Monitoring
|
||||||
|
|
||||||
|
Nothing collects metrics or logs, not even metrics-server (`kubectl top`
|
||||||
|
fails). The dqlite stalls were tracked down by hand from the API server's
|
||||||
|
metrics, and Velero's scheduled backups failed every night from August
|
||||||
|
until 2026-10-21 without anyone noticing.
|
||||||
|
|
||||||
|
- [x] A stopgap for the worst blind spots (2026-10-25,
|
||||||
|
`gatus/backup-check.yaml`): an hourly CronJob reads the API and pushes
|
||||||
|
four external endpoints to s2 (group "cluster"): the newest daily and
|
||||||
|
weekly Velero backup Completed and recent, every attached Longhorn
|
||||||
|
volume healthy with a backup younger than 26h, and no Vault pod sealed.
|
||||||
|
Gatus alerts through ntfy after two failed reports, or when no report
|
||||||
|
arrives for 3 hours. The token comes from an ESO Password generator.
|
||||||
|
Tested with made-up data for each failure (a Failed or old backup, a
|
||||||
|
degraded or unbacked-up volume, a sealed pod)
|
||||||
|
- [ ] After the second SSD (a time series database writes all the time
|
||||||
|
and would add to today's stalls): metrics-server, then Prometheus or
|
||||||
|
VictoriaMetrics with Grafana, alerting to ntfy. Loki later
|
||||||
|
- [ ] Alerts worth having: disk latency and fill, lease losses and
|
||||||
|
controller restarts, backups that don't complete, Longhorn volumes that
|
||||||
|
aren't healthy, Vault sealed
|
||||||
|
|
||||||
|
### 7. Limits, probes, network policies
|
||||||
|
|
||||||
|
Counted 2026-10-25 over the 73 running containers:
|
||||||
|
|
||||||
|
- 59 have no requests and 64 no memory limit, among them all of Argo CD,
|
||||||
|
authentik, cert-manager, ESO, mariadb-operator and Vault, and freshrss,
|
||||||
|
gitea, immich and serpbear. The node has 32 GB with 14.6 GB in use; one
|
||||||
|
runaway pod can take the node down, dqlite with it.
|
||||||
|
- No probes on gitea, freshrss, serpbear, authentik's LDAP outpost, the
|
||||||
|
ESO controller, dex and Argo CD's redis.
|
||||||
|
- NetworkPolicies exist only in argocd and longhorn-system. Public are
|
||||||
|
immich (every member, plus public share links), mbgwp (WordPress) and
|
||||||
|
authentik.
|
||||||
|
- Running as root as far as the securityContext tells (images not
|
||||||
|
checked): authentik, bolletjestrui, freshrss, gatus, gitea, immich,
|
||||||
|
ntfy, serpbear, mariadb-operator.
|
||||||
|
|
||||||
|
- [x] NetworkPolicies for immich, mbgwp and authentik
|
||||||
|
(`<namespace>/networkpolicy.yaml`, 2026-10-25). Before, WordPress could
|
||||||
|
reach the NAS's DSM, the router, the API server, Vault and immich. Now
|
||||||
|
each of the three reaches only its own pods, DNS and the internet
|
||||||
|
(without the private ranges), plus the API server for the CNPG
|
||||||
|
instances and authentik's worker. Ingress only from Envoy, the CNPG and
|
||||||
|
MariaDB operators, the node (kubelet probes) and, for the LDAP outpost,
|
||||||
|
from anywhere. Checked afterwards: the sites, the OIDC endpoints and
|
||||||
|
logins, Longhorn's login through Envoy, the LDAP NodePort, CNPG
|
||||||
|
healthy, a MariaDB backup and a share-sync run. The node's addresses
|
||||||
|
(192.168.1.204 and Calico's 10.1.7.64) are written out, so a new node
|
||||||
|
means new policies
|
||||||
|
- [x] The same for gitea, freshrss, serpbear, ntfy, actual-budget,
|
||||||
|
bolletjestrui and gatus (2026-10-25), plus gatus → ntfy in-cluster.
|
||||||
|
home-assistant has no pods (it points at 192.168.1.202). Vault now only
|
||||||
|
takes connections from ESO, Envoy (the UI) and the node (the injector's
|
||||||
|
webhook); its pods reach each other, DNS, the API server and the
|
||||||
|
internet. Checked: every host answers as before, gatus s2 16/16, the
|
||||||
|
OIDC logins of Vault and FreshRSS reach authentik, all 24
|
||||||
|
ExternalSecrets refresh, raft intact; Gitea no longer reaches the NAS
|
||||||
|
or Vault, and a pod in `default` no longer reaches Vault or Gitea
|
||||||
|
- [ ] The infrastructure namespaces (cert-manager, cnpg-system,
|
||||||
|
external-secrets, envoy-gateway-system, velero, mariadb-system, dns,
|
||||||
|
default) still have none; `default` can reach the LAN and everything
|
||||||
|
without a policy
|
||||||
|
- [ ] Requests and memory limits, the big ones first (immich, authentik,
|
||||||
|
Argo CD's repo-server)
|
||||||
|
- [ ] Probes
|
||||||
|
- [ ] Non-root where the image allows it
|
||||||
|
|
||||||
|
### 8. Renovate and CI
|
||||||
|
|
||||||
|
10 Renovate PRs are open, the oldest from February, and nothing checks a
|
||||||
|
PR before Argo CD applies it.
|
||||||
|
|
||||||
|
- [ ] A GitHub Actions workflow that renders the Helm apps and runs
|
||||||
|
kubeconform on every PR, with conftest for this repo's rules
|
||||||
|
- [ ] Automerge patch versions of low-risk apps once that runs
|
||||||
|
|
||||||
|
### 9. One node
|
||||||
|
|
||||||
|
One node, one SSD, one replica per volume. The second SSD takes away the
|
||||||
|
disk contention; a second and third node (nix-infra, like node001) is the
|
||||||
|
step after that, if downtime starts to matter.
|
||||||
|
|
||||||
|
### 10. Repo and cluster cleanup
|
||||||
|
|
||||||
|
- [x] Done 2026-10-25: `disable/` and the `.DS_Store` files, now in
|
||||||
|
`.gitignore`; `secrets/hensenio-smtp.yaml` and `velero/minio-es.yaml`
|
||||||
|
(commented out in full), and with the first the `secrets-folder` app;
|
||||||
|
`backup-testing/test-velero-backup.sh.bak` (an older copy); the 15
|
||||||
|
completed kopia maintenance jobs of repositories that no longer exist
|
||||||
|
(nextcloud, owntracks, sogo, sogo-mail, ttrss)
|
||||||
|
- [ ] `kubectl delete namespace secrets`, empty since its app went
|
||||||
|
- [ ] 41 Failed Velero backups: the dailies from 2026-09-26 to 10-21 and
|
||||||
|
the weeklies since 08-02. Velero doesn't restore from a Failed backup;
|
||||||
|
left alone they expire by 2027-01-16.
|
||||||
|
`kubectl -n velero exec deploy/velero -c velero -- /velero backup delete $(kubectl -n velero get backups.velero.io -o json | jq -r '.items[] | select(.status.phase=="Failed") | .metadata.name' | xargs) --confirm`
|
||||||
|
- [ ] The mariadb-operator Backup `mbgwp-pre-migration-backup`, a dump
|
||||||
|
from 2026-02-22 on its own 10 GiB volume (which has a Longhorn backup
|
||||||
|
from 2026-10-22). The PVC has no owner, so it goes separately:
|
||||||
|
`kubectl -n mbgwp delete backups.k8s.mariadb.com mbgwp-pre-migration-backup && kubectl -n mbgwp delete pvc mbgwp-pre-migration-backup`
|
||||||
|
- [ ] 50 of the 70 Longhorn backup volumes on the NAS belong to volumes
|
||||||
|
that no longer exist. Keep the final backups of 2026-10-22 (ghost-blog,
|
||||||
|
stalwart, whisper). The rest are from apps removed long ago: nextcloud
|
||||||
|
(2025-03), owntracks, traccar, ttrss, whisper-priv, whisper-webui,
|
||||||
|
ghost, sogo, sogo-mail, outrun-game, the old mbg-wordpress and
|
||||||
|
mbgwp-helm volumes, immich's database before CNPG. Decide per app and
|
||||||
|
delete in the Longhorn UI (Backup)
|
||||||
|
- [ ] opti1, without sudo: `rm ~/calico-3.29.3.yaml ~/cni-3.25.1-backup.yaml ~/hosts.toml.docker.io.bak ~/dqlite-vacuum`,
|
||||||
|
rollback material for steps that can't be undone any more (the tool's
|
||||||
|
source is in tools/)
|
||||||
|
- [ ] opti1, with sudo: `/root/backend-*` and `/root/dqlite-pre-*` (from
|
||||||
|
before 1.33, 1.34 and the VACUUM; 1.34 migrated the schema, so none of
|
||||||
|
them fits 1.36) and `~/containerd.bak` (root-owned, from 2025-02)
|
||||||
|
- [ ] `~/cadir` on opti1 holds a CA key and certificate from 2026-03-21
|
||||||
|
that are not microk8s's current CA. Find out whether anything still
|
||||||
|
trusts it; if not, delete it
|
||||||
|
|
||||||
|
## wishlist.txt (2026-01-25)
|
||||||
|
|
||||||
|
- Done: Renovate; secrets out of git (Vault and ESO); SSO through authentik
|
||||||
|
(Argo CD, Vault, Longhorn, FreshRSS, Actual, mail, immich); Gitea as a
|
||||||
|
GitHub alternative; nix-infra for node001; most of the repo cleanup.
|
||||||
|
- Partly: network policies, probes, requests and limits, non-root (7);
|
||||||
|
backup testing (a script, not scheduled; 2); understanding Longhorn
|
||||||
|
(docs/longhorn-second-ssd.md).
|
||||||
|
- Open: Prometheus, Grafana, Loki, Alertmanager and which alerts (6);
|
||||||
|
conftest and CI/CD (8); Trivy; a staging environment; an image registry;
|
||||||
|
three nodes and opti1 on nix-infra (9); less manual work per new app.
|
||||||
|
kube-hunter is no longer developed; the Trivy Operator or kube-bench
|
||||||
|
cover that ground.
|
||||||
|
- Dropped: shrinking the immich PVC to 200 GB. Longhorn can't shrink a
|
||||||
|
volume, and with personal accounts the library will grow. Deleting stale
|
||||||
|
snapshots (docs/longhorn-second-ssd.md) frees more.
|
||||||
@@ -267,9 +267,24 @@ whatever time of day the hourly schedule had drifted to.
|
|||||||
CD app), the CNPG operator (`additionalArgs`) and Envoy Gateway
|
CD app), the CNPG operator (`additionalArgs`) and Envoy Gateway
|
||||||
(`provider.kubernetes.leaderElection`). Longhorn's CSI sidecars and
|
(`provider.kubernetes.leaderElection`). Longhorn's CSI sidecars and
|
||||||
global manager have no setting for it
|
global manager have no setting for it
|
||||||
- [ ] `~/iosample.sh` on opti1 logs sda's counters and io pressure every 2s
|
- [x] `~/iosample.sh` on opti1 logs sda's counters and io pressure every 2s
|
||||||
for 26h (from 2026-10-24 12:35 UTC) into `~/iosample.log`, to put
|
for 26h (from 2026-10-24 12:35 UTC) into `~/iosample.log`, to put
|
||||||
numbers on the disk latency during tonight's backups
|
numbers on the disk latency during tonight's backups. Read on 2026-10-25
|
||||||
|
at 02:35 UTC, per 10 minutes:
|
||||||
|
- Quiet: ~50 writes/s, 0.5-2 ms per write on average, "full" IO
|
||||||
|
pressure under 15%
|
||||||
|
- 14:30-23:15: 250-440 writes/s and 200-400 reads/s, 6-20 ms per write,
|
||||||
|
pressure 45-75%. Longhorn merging immich-claim's deleted snapshots
|
||||||
|
(docs/longhorn-second-ssd.md)
|
||||||
|
- Longhorn backups 00:00-00:15: up to 1,220 reads/s, pressure 69%; the
|
||||||
|
Velero backup at 01:00 was incremental and short (pressure 29%)
|
||||||
|
- The authentik and Vault upgrades 01:20-01:40 (image pulls,
|
||||||
|
migrations): 19-33 ms per write, single 2s windows near 2 s per write,
|
||||||
|
pressure 78%. The CSI sidecars and Longhorn's global manager lost
|
||||||
|
their leases at 01:31
|
||||||
|
Ubuntu's sysstat already keeps per-device history every 10 minutes
|
||||||
|
without a script: `sar -d -f /var/log/sysstat/sa24` showed sda at
|
||||||
|
~26 MB/s read and ~27 MB/s written, 9 ms await, during the merging
|
||||||
- [ ] Structural fix: a second SSD, see docs/longhorn-second-ssd.md. The OptiPlex 5060 (Micro, going by the
|
- [ ] Structural fix: a second SSD, see docs/longhorn-second-ssd.md. The OptiPlex 5060 (Micro, going by the
|
||||||
i5-8600T) has an M.2 slot that takes NVMe, and no NVMe device is
|
i5-8600T) has an M.2 slot that takes NVMe, and no NVMe device is
|
||||||
present; the Apacer AS350 is a DRAM-less 2.5" SATA SSD. An NVMe SSD with
|
present; the Apacer AS350 is a DRAM-less 2.5" SATA SSD. An NVMe SSD with
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
# FreshRSS faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces.
|
||||||
|
# Feeds and the OIDC login (authentik's public address) go out over
|
||||||
|
# the internet.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
# Checks what Gatus can't see from outside and reports it to Gatus as
|
||||||
|
# external endpoints (group "cluster" on s2), which alert through ntfy when
|
||||||
|
# a check fails or when no report arrives (heartbeat). Velero's scheduled
|
||||||
|
# backups failed every night from August to 2026-10-21 without anyone
|
||||||
|
# noticing; a sealed Vault after a reboot looks fine from outside too.
|
||||||
|
#
|
||||||
|
# Every hour, from the Kubernetes API:
|
||||||
|
# - velero-daily / velero-weekly: the newest finished backup of the
|
||||||
|
# schedule is Completed and younger than 26h / 8 days
|
||||||
|
# - longhorn: every attached volume is healthy and has a completed backup
|
||||||
|
# younger than 26h (detached volumes are skipped; the recurring job skips
|
||||||
|
# them too)
|
||||||
|
# - vault-unsealed: all Vault pods report vault-sealed=false
|
||||||
|
#
|
||||||
|
# The token is generated by ESO and never leaves the cluster; Gatus reads
|
||||||
|
# it from the same Secret.
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
metadata:
|
||||||
|
name: backup-check-token
|
||||||
|
spec:
|
||||||
|
length: 40
|
||||||
|
digits: 10
|
||||||
|
symbols: 0
|
||||||
|
noUpper: false
|
||||||
|
allowRepeat: true
|
||||||
|
secretKeys:
|
||||||
|
- BACKUP_CHECK_TOKEN
|
||||||
|
---
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: backup-check-token
|
||||||
|
spec:
|
||||||
|
# Generate once; a new token would need a Gatus restart.
|
||||||
|
refreshInterval: "0"
|
||||||
|
target:
|
||||||
|
name: backup-check-token
|
||||||
|
creationPolicy: Owner
|
||||||
|
dataFrom:
|
||||||
|
- sourceRef:
|
||||||
|
generatorRef:
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
name: backup-check-token
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
rules:
|
||||||
|
- apiGroups: ["velero.io"]
|
||||||
|
resources: ["backups"]
|
||||||
|
verbs: ["list"]
|
||||||
|
- apiGroups: ["longhorn.io"]
|
||||||
|
resources: ["backups", "volumes"]
|
||||||
|
verbs: ["list"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods"]
|
||||||
|
verbs: ["list"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: backup-check
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: backup-check
|
||||||
|
namespace: gatus
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
data:
|
||||||
|
check.py: |
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
SA = "/var/run/secrets/kubernetes.io/serviceaccount"
|
||||||
|
API = "https://kubernetes.default.svc"
|
||||||
|
GATUS = os.environ.get("GATUS_URL", "http://gatus:8080")
|
||||||
|
TOKEN = os.environ["BACKUP_CHECK_TOKEN"]
|
||||||
|
NOW = dt.datetime.now(dt.timezone.utc)
|
||||||
|
FINISHED = {"Completed", "PartiallyFailed", "Failed", "FailedValidation"}
|
||||||
|
|
||||||
|
with open(f"{SA}/token") as f:
|
||||||
|
sa_token = f.read()
|
||||||
|
api_ctx = ssl.create_default_context(cafile=f"{SA}/ca.crt")
|
||||||
|
|
||||||
|
|
||||||
|
def items(path):
|
||||||
|
req = urllib.request.Request(API + path, headers={"Authorization": f"Bearer {sa_token}"})
|
||||||
|
with urllib.request.urlopen(req, context=api_ctx, timeout=30) as r:
|
||||||
|
return json.load(r)["items"]
|
||||||
|
|
||||||
|
|
||||||
|
def parse(ts):
|
||||||
|
return dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||||
|
|
||||||
|
|
||||||
|
def age(ts):
|
||||||
|
hours = (NOW - parse(ts)).total_seconds() / 3600
|
||||||
|
return f"{hours:.0f}h"
|
||||||
|
|
||||||
|
|
||||||
|
def velero(schedule, max_hours):
|
||||||
|
backups = [
|
||||||
|
b for b in items("/apis/velero.io/v1/namespaces/velero/backups")
|
||||||
|
if b["metadata"].get("labels", {}).get("velero.io/schedule-name") == schedule
|
||||||
|
and b.get("status", {}).get("phase") in FINISHED
|
||||||
|
]
|
||||||
|
if not backups:
|
||||||
|
return f"no finished {schedule} backup"
|
||||||
|
newest = max(backups, key=lambda b: b["status"].get("startTimestamp", ""))
|
||||||
|
status = newest["status"]
|
||||||
|
name = newest["metadata"]["name"]
|
||||||
|
if status["phase"] != "Completed":
|
||||||
|
return f"{name} is {status['phase']} ({status.get('errors', 0)} errors)"
|
||||||
|
done = status.get("completionTimestamp")
|
||||||
|
if not done or (NOW - parse(done)).total_seconds() > max_hours * 3600:
|
||||||
|
return f"newest {schedule} backup {name} is {age(done or status['startTimestamp'])} old"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def longhorn(max_hours):
|
||||||
|
newest = {}
|
||||||
|
for b in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/backups"):
|
||||||
|
s = b.get("status", {})
|
||||||
|
if s.get("state") == "Completed" and s.get("backupCreatedAt"):
|
||||||
|
v = s.get("volumeName")
|
||||||
|
newest[v] = max(newest.get(v, ""), s["backupCreatedAt"])
|
||||||
|
problems = []
|
||||||
|
for v in items("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes"):
|
||||||
|
s = v.get("status", {})
|
||||||
|
if s.get("state") != "attached":
|
||||||
|
continue
|
||||||
|
name = v["metadata"]["name"]
|
||||||
|
pvc = s.get("kubernetesStatus", {}).get("pvcName") or name
|
||||||
|
if s.get("robustness") != "healthy":
|
||||||
|
problems.append(f"{pvc} is {s.get('robustness')}")
|
||||||
|
last = newest.get(name)
|
||||||
|
if not last:
|
||||||
|
problems.append(f"{pvc} has no backup")
|
||||||
|
elif (NOW - parse(last)).total_seconds() > max_hours * 3600:
|
||||||
|
problems.append(f"{pvc} last backed up {age(last)} ago")
|
||||||
|
return "; ".join(problems) or None
|
||||||
|
|
||||||
|
|
||||||
|
def vault():
|
||||||
|
pods = items("/api/v1/namespaces/vault/pods?labelSelector=component%3Dserver")
|
||||||
|
if not pods:
|
||||||
|
return "no Vault pods"
|
||||||
|
sealed = [p["metadata"]["name"] for p in pods
|
||||||
|
if p["metadata"].get("labels", {}).get("vault-sealed") != "false"]
|
||||||
|
return f"sealed: {', '.join(sealed)}" if sealed else None
|
||||||
|
|
||||||
|
|
||||||
|
def push(key, error):
|
||||||
|
query = {"success": "false" if error else "true"}
|
||||||
|
if error:
|
||||||
|
query["error"] = error[:500]
|
||||||
|
url = f"{GATUS}/api/v1/endpoints/{key}/external?{urllib.parse.urlencode(query)}"
|
||||||
|
req = urllib.request.Request(url, method="POST", headers={"Authorization": f"Bearer {TOKEN}"})
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
r.read()
|
||||||
|
|
||||||
|
|
||||||
|
checks = {
|
||||||
|
"cluster_velero-daily": lambda: velero("daily-backup", 26),
|
||||||
|
"cluster_velero-weekly": lambda: velero("weekly-backup", 8 * 24 + 2),
|
||||||
|
"cluster_longhorn": lambda: longhorn(26),
|
||||||
|
"cluster_vault-unsealed": vault,
|
||||||
|
}
|
||||||
|
failed_pushes = 0
|
||||||
|
for key, check in checks.items():
|
||||||
|
try:
|
||||||
|
error = check()
|
||||||
|
except Exception as e: # report a broken check as a failure
|
||||||
|
error = f"check failed: {e}"
|
||||||
|
print(f"{key}: {error or 'ok'}", flush=True)
|
||||||
|
try:
|
||||||
|
push(key, error)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"{key}: push to Gatus failed: {e}", flush=True)
|
||||||
|
failed_pushes += 1
|
||||||
|
sys.exit(1 if failed_pushes else 0)
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: backup-check
|
||||||
|
spec:
|
||||||
|
schedule: "17 * * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
activeDeadlineSeconds: 300
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: backup-check
|
||||||
|
spec:
|
||||||
|
serviceAccountName: backup-check
|
||||||
|
restartPolicy: Never
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 65534
|
||||||
|
runAsGroup: 65534
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: check
|
||||||
|
image: python:3.13.16-alpine3.24
|
||||||
|
command: ["python", "/script/check.py"]
|
||||||
|
env:
|
||||||
|
- name: PYTHONDONTWRITEBYTECODE
|
||||||
|
value: "1"
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: backup-check-token
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /script
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: backup-check
|
||||||
@@ -136,3 +136,36 @@ endpoints:
|
|||||||
- "[STATUS] == 200"
|
- "[STATUS] == 200"
|
||||||
- "[BODY].status == UP"
|
- "[BODY].status == UP"
|
||||||
alerts: *ntfy
|
alerts: *ntfy
|
||||||
|
|
||||||
|
# Pushed every hour by the backup-check CronJob (backup-check.yaml), which
|
||||||
|
# looks at what can't be seen from outside. s2 alerts on these itself: s1
|
||||||
|
# has no view into the cluster.
|
||||||
|
external-endpoints:
|
||||||
|
- name: velero-daily
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: &cluster-alert
|
||||||
|
- type: ntfy
|
||||||
|
failure-threshold: 2
|
||||||
|
success-threshold: 1
|
||||||
|
send-on-resolved: true
|
||||||
|
- name: velero-weekly
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: *cluster-alert
|
||||||
|
- name: longhorn
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: *cluster-alert
|
||||||
|
- name: vault-unsealed
|
||||||
|
group: cluster
|
||||||
|
token: ${BACKUP_CHECK_TOKEN}
|
||||||
|
heartbeat:
|
||||||
|
interval: 3h
|
||||||
|
alerts: *cluster-alert
|
||||||
@@ -25,6 +25,9 @@ spec:
|
|||||||
envFrom:
|
envFrom:
|
||||||
- secretRef:
|
- secretRef:
|
||||||
name: gatus-ntfy
|
name: gatus-ntfy
|
||||||
|
# Token for the external endpoints, see backup-check.yaml
|
||||||
|
- secretRef:
|
||||||
|
name: backup-check-token
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8080
|
- containerPort: 8080
|
||||||
protocol: TCP
|
protocol: TCP
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ resources:
|
|||||||
- gatus-service.yaml
|
- gatus-service.yaml
|
||||||
- certificate.yaml
|
- certificate.yaml
|
||||||
- httproute.yaml
|
- httproute.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
|
- backup-check.yaml
|
||||||
|
|
||||||
configMapGenerator:
|
configMapGenerator:
|
||||||
- name: gatus-config
|
- name: gatus-config
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# Gatus faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces.
|
||||||
|
# Its checks go to public addresses; alerts go to ntfy in-cluster.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8080
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: ntfy
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: ntfy
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
# backup-check.yaml reads backups, volumes and Vault's pods from the API
|
||||||
|
# server (the node, port 16443).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: backup-check-to-api
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: backup-check
|
||||||
|
policyTypes:
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# Gitea faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces.
|
||||||
|
# Mirrors and webhooks go out over the internet.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 3000
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -27,7 +27,7 @@ spec:
|
|||||||
# immich_role and immich_quota claims (authentik/blueprint-scouting.yaml).
|
# immich_role and immich_quota claims (authentik/blueprint-scouting.yaml).
|
||||||
oauth:
|
oauth:
|
||||||
enabled: true
|
enabled: true
|
||||||
issuerUrl: https://authentik.rubenhensen.nl/application/o/immich/
|
issuerUrl: https://auth.scoutingmbg.nl/application/o/immich/
|
||||||
clientId: immich
|
clientId: immich
|
||||||
clientSecret: "{{ .client_secret }}"
|
clientSecret: "{{ .client_secret }}"
|
||||||
scope: openid email profile immich
|
scope: openid email profile immich
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
# immich faces the internet (every member of the group logs in, and shared
|
||||||
|
# links are public), so its pods only reach each other, DNS and the
|
||||||
|
# internet: not the LAN (NAS, router), the Kubernetes API or other
|
||||||
|
# namespaces. The OIDC login with authentik goes out through its public
|
||||||
|
# address. Ingress comes from Envoy, the CNPG operator and the node itself
|
||||||
|
# (kubelet probes; 10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
# The internet: authentik's public address, the ML models from Hugging
|
||||||
|
# Face.
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: server-from-envoy
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/instance: helm-immich
|
||||||
|
app.kubernetes.io/name: server
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 2283
|
||||||
|
---
|
||||||
|
# The CNPG operator checks each instance on its status port and connects to
|
||||||
|
# PostgreSQL; the instance manager inside each pod watches its Cluster
|
||||||
|
# through the API server (the node, port 16443).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: postgres-cnpg
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: cnpg.io/cluster
|
||||||
|
operator: Exists
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: cnpg-system
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
- port: 5432
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
# WordPress faces the internet, so if it is ever taken over it should only
|
||||||
|
# reach its own database, DNS and the internet: not the LAN (NAS, router),
|
||||||
|
# the Kubernetes API or other namespaces. Ingress comes from Envoy, the
|
||||||
|
# mariadb-operator and the node itself (kubelet probes; 10.1.7.64 is the
|
||||||
|
# node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
# The internet, which includes this site's own public address
|
||||||
|
# (wp-cron's loopback requests go out and come back through Envoy).
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: wordpress-from-envoy
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: wordpress
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: mariadb-from-operator
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: mariadb
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: mariadb-system
|
||||||
|
ports:
|
||||||
|
- port: 3306
|
||||||
@@ -20,6 +20,8 @@ let
|
|||||||
"git.rubenhensen.nl"
|
"git.rubenhensen.nl"
|
||||||
"vic.rubenhensen.nl"
|
"vic.rubenhensen.nl"
|
||||||
# The group's DNS points these at rubenhensen.nl, i.e. here
|
# The group's DNS points these at rubenhensen.nl, i.e. here
|
||||||
|
"scoutingmbg.nl"
|
||||||
|
"www.scoutingmbg.nl"
|
||||||
"foto.scoutingmbg.nl"
|
"foto.scoutingmbg.nl"
|
||||||
"auth.scoutingmbg.nl"
|
"auth.scoutingmbg.nl"
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -10,3 +10,4 @@ resources:
|
|||||||
- ntfy-service.yaml
|
- ntfy-service.yaml
|
||||||
- certificate.yaml
|
- certificate.yaml
|
||||||
- httproute.yaml
|
- httproute.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# ntfy faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces.
|
||||||
|
# Push notifications go out over the internet.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
# Gatus sends its alerts to ntfy in-cluster.
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: gatus
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: gatus
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: ntfy
|
- name: ntfy
|
||||||
image: binwiederhier/ntfy:v2.28.0
|
image: binwiederhier/ntfy:v2.34.0
|
||||||
args: ["serve"]
|
args: ["serve"]
|
||||||
env:
|
env:
|
||||||
- name: NTFY_BASE_URL
|
- name: NTFY_BASE_URL
|
||||||
|
|||||||
+2
-1
@@ -1,7 +1,8 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||||
"extends": [
|
"extends": [
|
||||||
"config:recommended"
|
"config:recommended",
|
||||||
|
"customManagers:githubActionsVersions"
|
||||||
],
|
],
|
||||||
"argocd": {
|
"argocd": {
|
||||||
"managerFilePatterns": [
|
"managerFilePatterns": [
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
# apiVersion: external-secrets.io/v1beta1
|
|
||||||
# kind: ExternalSecret
|
|
||||||
# metadata:
|
|
||||||
# name: hensenio-smtp-secrets
|
|
||||||
# namespace: blog
|
|
||||||
# spec:
|
|
||||||
# refreshInterval: "15m"
|
|
||||||
# secretStoreRef:
|
|
||||||
# name: vault-backend
|
|
||||||
# kind: ClusterSecretStore
|
|
||||||
# target:
|
|
||||||
# name: hensenio-smtp
|
|
||||||
# creationPolicy: Owner
|
|
||||||
# data:
|
|
||||||
# - secretKey: smtp-password
|
|
||||||
# remoteRef:
|
|
||||||
# key: kv/hensenio/smtp
|
|
||||||
# property: password
|
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# SerpBear faces the internet through Envoy, so its pods only reach each
|
||||||
|
# other, DNS and the internet: not the LAN (NAS, router), the Kubernetes
|
||||||
|
# API or other namespaces. Its scraper APIs are on the internet.
|
||||||
|
# Ingress comes from Envoy and the node itself (kubelet probes; 10.1.7.64
|
||||||
|
# is the node's Calico VXLAN address, projectcalico.org/IPv4VXLANTunnelAddr).
|
||||||
|
# Same layout as mbgwp/networkpolicy.yaml.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
ports:
|
||||||
|
- port: 3000
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Render every Argo CD app in this repo the way Argo CD does and validate
|
||||||
|
the output with kubeconform, so a broken manifest or a chart bump that no
|
||||||
|
longer renders fails the PR instead of the sync.
|
||||||
|
|
||||||
|
Run from anywhere; needs helm, kustomize, kubeconform, git and PyYAML.
|
||||||
|
The apps/ chart is rendered first, then every Application found in it and
|
||||||
|
in the app folders. Sources:
|
||||||
|
- a chart: helm template with the Application's values, value files and
|
||||||
|
parameters
|
||||||
|
- a path in this repo: helm template (Chart.yaml), kustomize build
|
||||||
|
(kustomization.yaml) or the folder's YAML files (directory include and
|
||||||
|
exclude)
|
||||||
|
- a path in another git repository: cloned at targetRevision, then the
|
||||||
|
same, with the Application's kustomize patches applied
|
||||||
|
Every YAML file in the repo is also checked for duplicate keys, which
|
||||||
|
YAML parsers resolve silently by keeping the last one.
|
||||||
|
"""
|
||||||
|
import fnmatch
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||||
|
OWN_REPOS = {"https://github.com/rubenhensen/k8scd.git", "https://github.com/rubenhensen/k8scd"}
|
||||||
|
SKIP_DIRS = {".git", ".github", "nix-infra-machine", "docs", "tools", "backup-testing"}
|
||||||
|
# Helm's view of the cluster; the JSON schemas only go up to 1.34.
|
||||||
|
KUBE_VERSION = "1.36.0"
|
||||||
|
SCHEMA_VERSION = "1.34.12"
|
||||||
|
CRD_SCHEMAS = ("https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/"
|
||||||
|
"{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json")
|
||||||
|
|
||||||
|
failures = []
|
||||||
|
|
||||||
|
|
||||||
|
def run(cmd, **kw):
|
||||||
|
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
|
||||||
|
|
||||||
|
|
||||||
|
class UniqueKeyLoader(yaml.SafeLoader):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def construct_mapping(loader, node, deep=False):
|
||||||
|
keys = set()
|
||||||
|
for key_node, _ in node.value:
|
||||||
|
key = loader.construct_object(key_node, deep=deep)
|
||||||
|
if key in keys:
|
||||||
|
raise yaml.constructor.ConstructorError(
|
||||||
|
None, None, f"duplicate key {key!r}", key_node.start_mark)
|
||||||
|
keys.add(key)
|
||||||
|
return loader.construct_mapping(node, deep)
|
||||||
|
|
||||||
|
|
||||||
|
UniqueKeyLoader.add_constructor(
|
||||||
|
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
|
||||||
|
|
||||||
|
|
||||||
|
def yaml_files():
|
||||||
|
for path in sorted(ROOT.rglob("*.y*ml")):
|
||||||
|
rel = path.relative_to(ROOT)
|
||||||
|
if rel.parts[0] in SKIP_DIRS or path.suffix not in (".yaml", ".yml"):
|
||||||
|
continue
|
||||||
|
yield path
|
||||||
|
|
||||||
|
|
||||||
|
def check_duplicate_keys():
|
||||||
|
for path in yaml_files():
|
||||||
|
if path.parent.name == "templates": # Helm templates aren't YAML yet
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
list(yaml.load_all(path.read_text(), Loader=UniqueKeyLoader))
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
failures.append(f"{path.relative_to(ROOT)}: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def validate(label, manifests):
|
||||||
|
"""kubeconform on rendered manifests; returns the parsed documents."""
|
||||||
|
try:
|
||||||
|
docs = [d for d in yaml.safe_load_all(manifests) if d]
|
||||||
|
except yaml.YAMLError:
|
||||||
|
docs = None # kubeconform reports where
|
||||||
|
if docs is not None:
|
||||||
|
# Argo CD skips documents that aren't objects, like a values file
|
||||||
|
# next to the manifests.
|
||||||
|
objects = [d for d in docs if isinstance(d, dict) and "kind" in d]
|
||||||
|
if len(objects) < len(docs):
|
||||||
|
print(f" {label}: skipping {len(docs) - len(objects)} document(s) without a kind")
|
||||||
|
manifests = yaml.safe_dump_all(objects, sort_keys=False)
|
||||||
|
result = subprocess.run(
|
||||||
|
["kubeconform", "-summary", "-output", "text", "-strict",
|
||||||
|
"-kubernetes-version", SCHEMA_VERSION, "-ignore-missing-schemas",
|
||||||
|
"-schema-location", "default", "-schema-location", CRD_SCHEMAS],
|
||||||
|
input=manifests, capture_output=True, text=True)
|
||||||
|
lines = [l for l in result.stdout.splitlines() if l.strip()]
|
||||||
|
summary = lines[-1] if lines else ""
|
||||||
|
print(f" {label}: {summary.replace('Summary: ', '')}")
|
||||||
|
if result.returncode != 0:
|
||||||
|
for l in lines[:-1]:
|
||||||
|
print(f" {l}")
|
||||||
|
failures.append(f"{label}: {summary or result.stderr.strip()}")
|
||||||
|
return docs or []
|
||||||
|
|
||||||
|
|
||||||
|
def helm_template(name, namespace, chart_args, helm):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
args = ["helm", "template", helm.get("releaseName", name), *chart_args,
|
||||||
|
"--namespace", namespace, "--kube-version", KUBE_VERSION,
|
||||||
|
"--include-crds"]
|
||||||
|
if helm.get("values"):
|
||||||
|
values = pathlib.Path(tmp, "values.yaml")
|
||||||
|
values.write_text(helm["values"])
|
||||||
|
args += ["-f", str(values)]
|
||||||
|
if helm.get("valuesObject"):
|
||||||
|
values = pathlib.Path(tmp, "values-object.yaml")
|
||||||
|
values.write_text(yaml.safe_dump(helm["valuesObject"]))
|
||||||
|
args += ["-f", str(values)]
|
||||||
|
for vf in helm.get("valueFiles", []):
|
||||||
|
# $values/<path> points at this repo through a source with ref: values
|
||||||
|
args += ["-f", str(ROOT / vf.split("/", 1)[1]) if vf.startswith("$") else vf]
|
||||||
|
for p in helm.get("parameters", []):
|
||||||
|
flag = "--set-string" if p.get("forceString") else "--set"
|
||||||
|
args += [flag, f"{p['name']}={p['value']}"]
|
||||||
|
return run(args)
|
||||||
|
|
||||||
|
|
||||||
|
def directory(path, spec):
|
||||||
|
include = spec.get("include", "*.yaml")
|
||||||
|
exclude = spec.get("exclude", "")
|
||||||
|
files = sorted(path.rglob("*") if spec.get("recurse") else path.glob("*"))
|
||||||
|
docs = []
|
||||||
|
for f in files:
|
||||||
|
if f.suffix not in (".yaml", ".yml", ".json") or not f.is_file():
|
||||||
|
continue
|
||||||
|
name = f.name
|
||||||
|
if not any(fnmatch.fnmatch(name, g) for g in include.strip("{}").split(",")):
|
||||||
|
continue
|
||||||
|
if exclude and any(fnmatch.fnmatch(name, g) for g in exclude.strip("{}").split(",")):
|
||||||
|
continue
|
||||||
|
docs.append(f.read_text())
|
||||||
|
return "\n---\n".join(docs)
|
||||||
|
|
||||||
|
|
||||||
|
def kustomize(path, spec):
|
||||||
|
if not spec:
|
||||||
|
return run(["kustomize", "build", str(path)])
|
||||||
|
# The Application's kustomize options on top, as Argo CD does.
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
k = {"apiVersion": "kustomize.config.k8s.io/v1beta1", "kind": "Kustomization",
|
||||||
|
"resources": [os.path.relpath(path, tmp)]}
|
||||||
|
for key in ("patches", "images", "namespace", "commonLabels", "namePrefix", "nameSuffix"):
|
||||||
|
if key in spec:
|
||||||
|
k[key] = spec[key]
|
||||||
|
pathlib.Path(tmp, "kustomization.yaml").write_text(yaml.safe_dump(k))
|
||||||
|
return run(["kustomize", "build", "--load-restrictor", "LoadRestrictionsNone", tmp])
|
||||||
|
|
||||||
|
|
||||||
|
def render_path(app, src, base):
|
||||||
|
path = base / src.get("path", ".")
|
||||||
|
namespace = app["spec"]["destination"].get("namespace", "default")
|
||||||
|
if (path / "Chart.yaml").exists():
|
||||||
|
if (path / "Chart.lock").exists():
|
||||||
|
run(["helm", "dependency", "build", str(path)])
|
||||||
|
return helm_template(app["metadata"]["name"], namespace, [str(path)], src.get("helm", {}))
|
||||||
|
if (path / "kustomization.yaml").exists() or src.get("kustomize"):
|
||||||
|
return kustomize(path, src.get("kustomize"))
|
||||||
|
return directory(path, src.get("directory", {}))
|
||||||
|
|
||||||
|
|
||||||
|
def render(app, src, clones):
|
||||||
|
name = app["metadata"]["name"]
|
||||||
|
namespace = app["spec"]["destination"].get("namespace", "default")
|
||||||
|
repo = src["repoURL"]
|
||||||
|
if src.get("chart"):
|
||||||
|
if repo.startswith("http"):
|
||||||
|
chart_args = [src["chart"], "--repo", repo]
|
||||||
|
else:
|
||||||
|
chart_args = [f"oci://{repo.rstrip('/')}/{src['chart']}"]
|
||||||
|
chart_args += ["--version", src["targetRevision"]]
|
||||||
|
return helm_template(name, namespace, chart_args, src.get("helm", {}))
|
||||||
|
if src.get("ref") and "path" not in src:
|
||||||
|
return None # only provides value files
|
||||||
|
if repo in OWN_REPOS:
|
||||||
|
return render_path(app, src, ROOT)
|
||||||
|
key = (repo, src.get("targetRevision", "HEAD"))
|
||||||
|
if key not in clones:
|
||||||
|
tmp = tempfile.mkdtemp()
|
||||||
|
run(["git", "clone", "--quiet", "--depth", "1", "--branch", key[1], repo, tmp])
|
||||||
|
clones[key] = pathlib.Path(tmp)
|
||||||
|
return render_path(app, src, clones[key])
|
||||||
|
|
||||||
|
|
||||||
|
def applications(docs):
|
||||||
|
return [d for d in docs if isinstance(d, dict) and d.get("kind") == "Application"
|
||||||
|
and d.get("apiVersion", "").startswith("argoproj.io/")]
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print("Duplicate keys")
|
||||||
|
check_duplicate_keys()
|
||||||
|
|
||||||
|
print("Applications")
|
||||||
|
apps = applications(validate("apps (chart)", run(
|
||||||
|
["helm", "template", "apps", str(ROOT / "apps"), "--kube-version", KUBE_VERSION])))
|
||||||
|
for path in yaml_files():
|
||||||
|
if path.parts[len(ROOT.parts)] == "apps":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
apps += applications([d for d in yaml.safe_load_all(path.read_text()) if d])
|
||||||
|
except yaml.YAMLError:
|
||||||
|
pass # reported by the duplicate key check or by the app that uses it
|
||||||
|
|
||||||
|
clones = {}
|
||||||
|
seen = set()
|
||||||
|
for app in sorted(apps, key=lambda a: a["metadata"]["name"]):
|
||||||
|
name = app["metadata"]["name"]
|
||||||
|
if name in seen:
|
||||||
|
continue
|
||||||
|
seen.add(name)
|
||||||
|
spec = app["spec"]
|
||||||
|
sources = spec.get("sources") or [spec["source"]]
|
||||||
|
for i, src in enumerate(sources):
|
||||||
|
label = name if len(sources) == 1 else f"{name}[{i}]"
|
||||||
|
try:
|
||||||
|
manifests = render(app, src, clones)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f" {label}: render failed")
|
||||||
|
lines = []
|
||||||
|
for line in (e.stderr or e.stdout).strip().splitlines():
|
||||||
|
if not lines or line != lines[-1]: # Helm repeats schema errors
|
||||||
|
lines.append(line)
|
||||||
|
for line in lines[:20]:
|
||||||
|
print(f" {line}")
|
||||||
|
failures.append(f"{label}: render failed")
|
||||||
|
continue
|
||||||
|
if manifests is not None:
|
||||||
|
validate(label, manifests)
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
print(f"\n{len(failures)} problem(s):")
|
||||||
|
for f in failures:
|
||||||
|
print(f"- {f}")
|
||||||
|
sys.exit(1)
|
||||||
|
print("\nAll apps render and validate.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+74
@@ -0,0 +1,74 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Restart the Vault pods one at a time and unseal each one: the standbys
|
||||||
|
# first, the active node last, as HashiCorp's HA upgrade guide asks. The
|
||||||
|
# StatefulSet uses OnDelete, so this is how a new image from
|
||||||
|
# vault/overrides.yaml reaches the pods once Argo CD has synced it. Also
|
||||||
|
# useful after a reboot, minus the restarts: see "unseal only" below.
|
||||||
|
#
|
||||||
|
# Run it on a machine with kubectl access and the unseal key at hand:
|
||||||
|
# tools/vault-roll.sh 2.2.1
|
||||||
|
# It asks for the key once per pod (hidden input) and stores nothing.
|
||||||
|
# Stop at any point with Ctrl-C; pods already done stay on the new version.
|
||||||
|
#
|
||||||
|
# Unseal only (after a reboot, all three come up sealed):
|
||||||
|
# for p in vault-0 vault-1 vault-2; do kubectl -n vault exec -ti $p -- vault operator unseal; done
|
||||||
|
set -euo pipefail
|
||||||
|
ns=vault
|
||||||
|
target=${1:?usage: $0 <version the pods should run, e.g. 2.2.1>}
|
||||||
|
|
||||||
|
image=$(kubectl -n $ns get sts vault -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||||
|
if [[ $image != *:$target ]]; then
|
||||||
|
echo "The StatefulSet has $image, not $target. Sync the vault app first." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
pods() { kubectl -n $ns get pods -l "app.kubernetes.io/name=vault,component=server,vault-active=$1" -o name | sed 's|pod/||'; }
|
||||||
|
status() { kubectl -n $ns exec "$1" -- vault status -format=json 2>/dev/null || true; }
|
||||||
|
summary() {
|
||||||
|
status "$1" | jq -r --arg p "$1" '"\($p): \(.version), sealed \(.sealed), \(if .is_self then "active" else "standby" end), raft index \(.raft_committed_index)"'
|
||||||
|
}
|
||||||
|
|
||||||
|
roll() {
|
||||||
|
local pod=$1 rc
|
||||||
|
echo
|
||||||
|
echo "== $pod"
|
||||||
|
kubectl -n $ns delete pod "$pod" --wait=true
|
||||||
|
# vault status exits 0 when unsealed, 2 when sealed, 1 on errors (and
|
||||||
|
# kubectl exec exits 1 while the container isn't running yet).
|
||||||
|
while :; do
|
||||||
|
rc=0
|
||||||
|
kubectl -n $ns exec "$pod" -- vault status >/dev/null 2>&1 || rc=$?
|
||||||
|
[[ $rc -ne 1 ]] && break
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if [[ $rc -eq 2 ]]; then
|
||||||
|
kubectl -n $ns exec -ti "$pod" -- vault operator unseal
|
||||||
|
fi
|
||||||
|
kubectl -n $ns wait --for=condition=Ready "pod/$pod" --timeout=180s
|
||||||
|
summary "$pod"
|
||||||
|
local version
|
||||||
|
version=$(status "$pod" | jq -r .version)
|
||||||
|
if [[ $version != "$target" ]]; then
|
||||||
|
echo "$pod runs $version, expected $target. Stopping." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
active=$(pods true)
|
||||||
|
standbys=$(pods false)
|
||||||
|
if [[ $(echo "$active" | wc -w) -ne 1 || $(echo "$standbys" | wc -w) -ne 2 ]]; then
|
||||||
|
echo "Expected one active node and two standbys, found active: $active, standbys: $standbys" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Active: $active. Standbys: $(echo $standbys)."
|
||||||
|
|
||||||
|
for pod in $standbys; do
|
||||||
|
roll "$pod"
|
||||||
|
done
|
||||||
|
# The active node goes last. Deleting the pod shuts Vault down cleanly, so
|
||||||
|
# it hands the HA lock to an upgraded standby right away; no step-down.
|
||||||
|
roll "$active"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "== done"
|
||||||
|
for pod in vault-0 vault-1 vault-2; do summary "$pod"; done
|
||||||
@@ -33,7 +33,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: configure
|
- name: configure
|
||||||
image: hashicorp/vault:1.15
|
image: hashicorp/vault:2.2.1
|
||||||
env:
|
env:
|
||||||
- name: VAULT_ADDR
|
- name: VAULT_ADDR
|
||||||
value: "http://vault-active.vault.svc:8200"
|
value: "http://vault-active.vault.svc:8200"
|
||||||
@@ -51,7 +51,7 @@ spec:
|
|||||||
args:
|
args:
|
||||||
- |
|
- |
|
||||||
vault auth enable oidc 2>/dev/null || true
|
vault auth enable oidc 2>/dev/null || true
|
||||||
vault auth tune -listing_visibility=unauth oidc/
|
vault auth tune -listing-visibility=unauth oidc/
|
||||||
vault policy write admin - <<'POLICY'
|
vault policy write admin - <<'POLICY'
|
||||||
path "*" {
|
path "*" {
|
||||||
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
|
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Vault holds every secret, so only ESO, the UI through Envoy, and the
|
||||||
|
# node reach it (the API server calls the injector's webhook from there;
|
||||||
|
# 10.1.7.64 is the node's Calico VXLAN address,
|
||||||
|
# projectcalico.org/IPv4VXLANTunnelAddr). Nothing uses agent injection.
|
||||||
|
# The pods reach each other (raft on 8201, the OIDC job), DNS, the API
|
||||||
|
# server (service registration labels the active pod, the injector) and
|
||||||
|
# the internet (the OIDC login through authentik's public address).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
spec:
|
||||||
|
podSelector: {}
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector: {}
|
||||||
|
- from:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 10.1.7.64/32
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: envoy-gateway-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: envoy
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: external-secrets
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: external-secrets
|
||||||
|
ports:
|
||||||
|
- port: 8200
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector: {}
|
||||||
|
- to:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: kube-system
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kube-dns
|
||||||
|
ports:
|
||||||
|
- protocol: UDP
|
||||||
|
port: 53
|
||||||
|
- protocol: TCP
|
||||||
|
port: 53
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 192.168.1.204/32
|
||||||
|
ports:
|
||||||
|
- port: 16443
|
||||||
|
- to:
|
||||||
|
- ipBlock:
|
||||||
|
cidr: 0.0.0.0/0
|
||||||
|
except:
|
||||||
|
- 10.0.0.0/8
|
||||||
|
- 100.64.0.0/10
|
||||||
|
- 169.254.0.0/16
|
||||||
|
- 172.16.0.0/12
|
||||||
|
- 192.168.0.0/16
|
||||||
@@ -14,9 +14,11 @@ injector:
|
|||||||
server:
|
server:
|
||||||
# Pinned to the running version: the chart would otherwise pull its own
|
# Pinned to the running version: the chart would otherwise pull its own
|
||||||
# default on the next pod restart, and with Shamir unsealing a Vault
|
# default on the next pod restart, and with Shamir unsealing a Vault
|
||||||
# upgrade needs someone with the unseal keys at hand.
|
# upgrade needs someone with the unseal keys at hand. The StatefulSet
|
||||||
|
# uses OnDelete, so after a sync tools/vault-roll.sh <tag> restarts and
|
||||||
|
# unseals the pods one at a time.
|
||||||
image:
|
image:
|
||||||
tag: "1.17.2"
|
tag: "2.2.1"
|
||||||
dataStorage:
|
dataStorage:
|
||||||
storageClass: longhorn
|
storageClass: longhorn
|
||||||
standalone:
|
standalone:
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
# apiVersion: external-secrets.io/v1beta1
|
|
||||||
# kind: ExternalSecret
|
|
||||||
# metadata:
|
|
||||||
# name: velero-minio-credentials
|
|
||||||
# namespace: velero
|
|
||||||
# spec:
|
|
||||||
# refreshInterval: "15s"
|
|
||||||
# secretStoreRef:
|
|
||||||
# name: vault-backend
|
|
||||||
# kind: ClusterSecretStore
|
|
||||||
# target:
|
|
||||||
# name: velero-minio-credentials
|
|
||||||
# creationPolicy: Owner
|
|
||||||
# data:
|
|
||||||
# - secretKey: cloud
|
|
||||||
# remoteRef:
|
|
||||||
# key: velero/minio-credentials
|
|
||||||
# template:
|
|
||||||
# engine: gotmpl
|
|
||||||
# data: |
|
|
||||||
# [default]
|
|
||||||
# aws_access_key_id={{ .aws_access_key_id }}
|
|
||||||
# aws_secret_access_key={{ .aws_secret_access_key }}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
- staging env
|
|
||||||
- automated backup testing
|
|
||||||
- conftest policy as code voor clean gitops repo regels. Leg fouten vast zodat ze niet meer gebeuren.
|
|
||||||
- beter begrip van Longhorn en deletion policies enzo.
|
|
||||||
- minder handmatig werk als ik een app wil deployen?
|
|
||||||
- 3-node cluster
|
|
||||||
- CI/CD pipeline
|
|
||||||
- image registry
|
|
||||||
- centralized logging met loki en grafana
|
|
||||||
- metrics met prometheus en grafana
|
|
||||||
- alertmanager met goede alerts
|
|
||||||
- uitzoeken wat goede alerts zijn
|
|
||||||
- hardware management met behulp van die idempotem client ding, ben de naam kwijt.
|
|
||||||
Dan een nieuwe server opzetten compleet met dat,
|
|
||||||
en dan overgaan door het als een nieuwe node toe te voegen en de andere uit te zetten.
|
|
||||||
- authentik en alles met sso openid verbinden
|
|
||||||
- Renovate voor automatische updates zodat ik niet zo achter raak.
|
|
||||||
- Trivy vulnarability scanning (CVE scans)
|
|
||||||
- Network policies
|
|
||||||
- clean up repo
|
|
||||||
- remove hardcodes secrets.
|
|
||||||
- health probes to all services
|
|
||||||
- limits and requests for all services.
|
|
||||||
- non root security policy for all services.
|
|
||||||
- kube hunter network scans
|
|
||||||
- Draai mijn eigen github alternative
|
|
||||||
- Scale down pvc immich to 200gb.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Reference in new issue
Block a user