mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-16 14:29:09 +01:00
Give leader-elected controllers longer leases
The snapshot-controller (1,059 restarts), the CNPG operator and Envoy Gateway restart whenever a lease renewal takes more than 10 seconds, which happens when heavy writes stall dqlite on the node's SSD. Raise lease duration, renew deadline and retry period from 15s/10s/2s to 60s/40s/5s so they ride out stalls of up to ~40 seconds. With one replica, or a standby, a slower takeover costs nothing here.
This commit is contained in:
1 parent
9014898fc5
commit
858904384c
3 files changed
+35
No files matched your search
@@ -19,6 +19,11 @@ spec:
|
|||||||
# upgrades instead of restarting them. Both clusters have a
|
# upgrades instead of restarting them. Both clusters have a
|
||||||
# single instance, so a restart means downtime.
|
# single instance, so a restart means downtime.
|
||||||
ENABLE_INSTANCE_MANAGER_INPLACE_UPDATES: "true"
|
ENABLE_INSTANCE_MANAGER_INPLACE_UPDATES: "true"
|
||||||
|
# Ride out dqlite stalls of up to ~40s instead of restarting when a
|
||||||
|
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
|
||||||
|
additionalArgs:
|
||||||
|
- --leader-lease-duration=60
|
||||||
|
- --leader-renew-deadline=40
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: cnpg-system
|
namespace: cnpg-system
|
||||||
|
|||||||
@@ -13,6 +13,19 @@ spec:
|
|||||||
chart: gateway-helm
|
chart: gateway-helm
|
||||||
# v1.9 supports Kubernetes 1.33 to 1.36
|
# v1.9 supports Kubernetes 1.33 to 1.36
|
||||||
targetRevision: v1.9.2
|
targetRevision: v1.9.2
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
config:
|
||||||
|
envoyGateway:
|
||||||
|
provider:
|
||||||
|
kubernetes:
|
||||||
|
# Ride out dqlite stalls of up to ~40s instead of restarting
|
||||||
|
# when a lease renewal takes over 10s
|
||||||
|
# (docs/upgrade-k8s-1.33.md).
|
||||||
|
leaderElection:
|
||||||
|
leaseDuration: 60s
|
||||||
|
renewDeadline: 40s
|
||||||
|
retryPeriod: 5s
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: envoy-gateway-system
|
namespace: envoy-gateway-system
|
||||||
|
|||||||
@@ -9,6 +9,23 @@ spec:
|
|||||||
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
repoURL: https://github.com/kubernetes-csi/external-snapshotter.git
|
||||||
targetRevision: v8.2.1
|
targetRevision: v8.2.1
|
||||||
path: deploy/kubernetes/snapshot-controller
|
path: deploy/kubernetes/snapshot-controller
|
||||||
|
kustomize:
|
||||||
|
patches:
|
||||||
|
# Ride out dqlite stalls of up to ~40s instead of restarting when a
|
||||||
|
# lease renewal takes over 10s (docs/upgrade-k8s-1.33.md).
|
||||||
|
- target:
|
||||||
|
kind: Deployment
|
||||||
|
name: snapshot-controller
|
||||||
|
patch: |-
|
||||||
|
- op: add
|
||||||
|
path: /spec/template/spec/containers/0/args/-
|
||||||
|
value: --leader-election-lease-duration=60s
|
||||||
|
- op: add
|
||||||
|
path: /spec/template/spec/containers/0/args/-
|
||||||
|
value: --leader-election-renew-deadline=40s
|
||||||
|
- op: add
|
||||||
|
path: /spec/template/spec/containers/0/args/-
|
||||||
|
value: --leader-election-retry-period=5s
|
||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: kube-system
|
namespace: kube-system
|
||||||
|
|||||||
Reference in new issue
Block a user