From 858904384c7ad6540101a1b842ec8bc2ab8a64fc Mon Sep 17 00:00:00 2001 From: Ruben Hensen Date: Sat, 24 Oct 2026 14:43:38 +0200 Subject: [PATCH] Give leader-elected controllers longer leases The snapshot-controller (1,059 restarts), the CNPG operator and Envoy Gateway restart whenever a lease renewal takes more than 10 seconds, which happens when heavy writes stall dqlite on the node's SSD. Raise lease duration, renew deadline and retry period from 15s/10s/2s to 60s/40s/5s so they ride out stalls of up to ~40 seconds. With one replica, or a standby, a slower takeover costs nothing here. --- apps/templates/cloudnativepg-helm.yaml | 5 +++++ apps/templates/envoy-gateway-helm.yaml | 13 +++++++++++++ apps/templates/snapshot-controller-helm.yaml | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/apps/templates/cloudnativepg-helm.yaml b/apps/templates/cloudnativepg-helm.yaml index ff18dc5..a838b5e 100644 --- a/apps/templates/cloudnativepg-helm.yaml +++ b/apps/templates/cloudnativepg-helm.yaml @@ -19,6 +19,11 @@ spec: # upgrades instead of restarting them. Both clusters have a # single instance, so a restart means downtime. ENABLE_INSTANCE_MANAGER_INPLACE_UPDATES: "true" + # Ride out dqlite stalls of up to ~40s instead of restarting when a + # lease renewal takes over 10s (docs/upgrade-k8s-1.33.md). + additionalArgs: + - --leader-lease-duration=60 + - --leader-renew-deadline=40 destination: server: https://kubernetes.default.svc namespace: cnpg-system diff --git a/apps/templates/envoy-gateway-helm.yaml b/apps/templates/envoy-gateway-helm.yaml index 3522a7d..e5817bf 100644 --- a/apps/templates/envoy-gateway-helm.yaml +++ b/apps/templates/envoy-gateway-helm.yaml @@ -13,6 +13,19 @@ spec: chart: gateway-helm # v1.9 supports Kubernetes 1.33 to 1.36 targetRevision: v1.9.2 + helm: + values: | + config: + envoyGateway: + provider: + kubernetes: + # Ride out dqlite stalls of up to ~40s instead of restarting + # when a lease renewal takes over 10s + # (docs/upgrade-k8s-1.33.md). + leaderElection: + leaseDuration: 60s + renewDeadline: 40s + retryPeriod: 5s destination: server: https://kubernetes.default.svc namespace: envoy-gateway-system diff --git a/apps/templates/snapshot-controller-helm.yaml b/apps/templates/snapshot-controller-helm.yaml index 9581205..b60486e 100644 --- a/apps/templates/snapshot-controller-helm.yaml +++ b/apps/templates/snapshot-controller-helm.yaml @@ -9,6 +9,23 @@ spec: repoURL: https://github.com/kubernetes-csi/external-snapshotter.git targetRevision: v8.2.1 path: deploy/kubernetes/snapshot-controller + kustomize: + patches: + # Ride out dqlite stalls of up to ~40s instead of restarting when a + # lease renewal takes over 10s (docs/upgrade-k8s-1.33.md). + - target: + kind: Deployment + name: snapshot-controller + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --leader-election-lease-duration=60s + - op: add + path: /spec/template/spec/containers/0/args/- + value: --leader-election-renew-deadline=40s + - op: add + path: /spec/template/spec/containers/0/args/- + value: --leader-election-retry-period=5s destination: server: https://kubernetes.default.svc namespace: kube-system