Give leader-elected controllers longer leases

The snapshot-controller (1,059 restarts), the CNPG operator and Envoy
Gateway restart whenever a lease renewal takes more than 10 seconds,
which happens when heavy writes stall dqlite on the node's SSD. Raise
lease duration, renew deadline and retry period from 15s/10s/2s to
60s/40s/5s so they ride out stalls of up to ~40 seconds. With one
replica, or a standby, a slower takeover costs nothing here.
This commit is contained in:
Ruben Hensen committed 2026-10-24 14:43:38 +02:00
1 parent 9014898fc5
commit 858904384c
3 files changed
+35

No files matched your search

+5
View File
@@ -19,6 +19,11 @@ spec:
# upgrades instead of restarting them. Both clusters have a
# single instance, so a restart means downtime.
ENABLE_INSTANCE_MANAGER_INPLACE_UPDATES: "true"
# Ride out dqlite stalls of up to ~40s instead of restarting when a
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
additionalArgs:
- --leader-lease-duration=60
- --leader-renew-deadline=40
destination:
server: https://kubernetes.default.svc
namespace: cnpg-system
+13
View File
@@ -13,6 +13,19 @@ spec:
chart: gateway-helm
# v1.9 supports Kubernetes 1.33 to 1.36
targetRevision: v1.9.2
helm:
values: |
config:
envoyGateway:
provider:
kubernetes:
# Ride out dqlite stalls of up to ~40s instead of restarting
# when a lease renewal takes over 10s
# (docs/upgrade-k8s-1.33.md).
leaderElection:
leaseDuration: 60s
renewDeadline: 40s
retryPeriod: 5s
destination:
server: https://kubernetes.default.svc
namespace: envoy-gateway-system
@@ -9,6 +9,23 @@ spec:
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
targetRevision: v8.2.1
path: deploy/kubernetes/snapshot-controller
kustomize:
patches:
# Ride out dqlite stalls of up to ~40s instead of restarting when a
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
- target:
kind: Deployment
name: snapshot-controller
patch: |-
- op: add
path: /spec/template/spec/containers/0/args/-
value: --leader-election-lease-duration=60s
- op: add
path: /spec/template/spec/containers/0/args/-
value: --leader-election-renew-deadline=40s
- op: add
path: /spec/template/spec/containers/0/args/-
value: --leader-election-retry-period=5s
destination:
server: https://kubernetes.default.svc
namespace: kube-system