mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-15 14:29:09 +01:00
Give leader-elected controllers longer leases
The snapshot-controller (1,059 restarts), the CNPG operator and Envoy Gateway restart whenever a lease renewal takes more than 10 seconds, which happens when heavy writes stall dqlite on the node's SSD. Raise lease duration, renew deadline and retry period from 15s/10s/2s to 60s/40s/5s so they ride out stalls of up to ~40 seconds. With one replica, or a standby, a slower takeover costs nothing here.
This commit is contained in:
1 parent
9014898fc5
commit
858904384c
3 files changed
+35
No files matched your search
@@ -19,6 +19,11 @@ spec:
|
||||
# upgrades instead of restarting them. Both clusters have a
|
||||
# single instance, so a restart means downtime.
|
||||
ENABLE_INSTANCE_MANAGER_INPLACE_UPDATES: "true"
|
||||
# Ride out dqlite stalls of up to ~40s instead of restarting when a
|
||||
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
|
||||
additionalArgs:
|
||||
- --leader-lease-duration=60
|
||||
- --leader-renew-deadline=40
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cnpg-system
|
||||
|
||||
@@ -13,6 +13,19 @@ spec:
|
||||
chart: gateway-helm
|
||||
# v1.9 supports Kubernetes 1.33 to 1.36
|
||||
targetRevision: v1.9.2
|
||||
helm:
|
||||
values: |
|
||||
config:
|
||||
envoyGateway:
|
||||
provider:
|
||||
kubernetes:
|
||||
# Ride out dqlite stalls of up to ~40s instead of restarting
|
||||
# when a lease renewal takes over 10s
|
||||
# (docs/upgrade-k8s-1.33.md).
|
||||
leaderElection:
|
||||
leaseDuration: 60s
|
||||
renewDeadline: 40s
|
||||
retryPeriod: 5s
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: envoy-gateway-system
|
||||
|
||||
@@ -9,6 +9,23 @@ spec:
|
||||
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
||||
targetRevision: v8.2.1
|
||||
path: deploy/kubernetes/snapshot-controller
|
||||
kustomize:
|
||||
patches:
|
||||
# Ride out dqlite stalls of up to ~40s instead of restarting when a
|
||||
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: snapshot-controller
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/args/-
|
||||
value: --leader-election-lease-duration=60s
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/args/-
|
||||
value: --leader-election-renew-deadline=40s
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/args/-
|
||||
value: --leader-election-retry-period=5s
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kube-system
|
||||
|
||||
Reference in new issue
Block a user