From 5c79f454a528ee3a961818fdd44c14118df03f80 Mon Sep 17 00:00:00 2001 From: jannfis Date: Fri, 3 Apr 2020 22:03:31 +0200 Subject: [PATCH 1/2] Increase HAProxy check interval to prevent intermittent failures/state flapping --- .../ha/base/redis-ha/chart/upstream.yaml | 49 +++++++++---------- manifests/ha/base/redis-ha/generate.sh | 4 +- manifests/ha/install.yaml | 24 ++++----- manifests/ha/namespace-install.yaml | 24 ++++----- 4 files changed, 51 insertions(+), 50 deletions(-) diff --git a/manifests/ha/base/redis-ha/chart/upstream.yaml b/manifests/ha/base/redis-ha/chart/upstream.yaml index 74ff31922debf..5641982459e97 100644 --- a/manifests/ha/base/redis-ha/chart/upstream.yaml +++ b/manifests/ha/base/redis-ha/chart/upstream.yaml @@ -1,11 +1,10 @@ -# This is an auto-generated file. DO NOT EDIT --- # Source: redis-ha/charts/redis-ha/templates/redis-ha-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: argocd-redis-ha-configmap - namespace: default + namespace: argocd labels: heritage: Tiller release: argocd @@ -155,9 +154,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE0 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_1 mode tcp @@ -169,9 +168,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE1 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_2 mode tcp @@ -183,9 +182,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE2 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # decide redis backend to use #master @@ -204,11 +203,11 @@ data: tcp-check send QUIT\r\n tcp-check expect string +OK use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 } - server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1 + server R0 argocd-redis-ha-announce-0:6379 check inter 3s fall 1 rise 1 use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 } - server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1 + server R1 argocd-redis-ha-announce-1:6379 check inter 3s fall 1 rise 1 use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 } - server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1 + server R2 argocd-redis-ha-announce-2:6379 check inter 3s fall 1 rise 1 haproxy_init.sh: | HAPROXY_CONF=/data/haproxy.cfg cp /readonly/haproxy.cfg "$HAPROXY_CONF" @@ -268,7 +267,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: argocd-redis-ha - namespace: default + namespace: argocd labels: heritage: Tiller release: argocd @@ -282,7 +281,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: argocd-redis-ha-haproxy - namespace: default + namespace: argocd labels: heritage: Tiller release: argocd @@ -296,7 +295,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: argocd-redis-ha - namespace: default + namespace: argocd labels: heritage: Tiller release: argocd @@ -317,7 +316,7 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: argocd-redis-ha - namespace: default + namespace: argocd labels: heritage: Tiller release: argocd @@ -339,7 +338,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-announce-0 - namespace: default + namespace: argocd labels: app: redis-ha heritage: "Tiller" @@ -368,7 +367,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-announce-1 - namespace: default + namespace: argocd labels: app: redis-ha heritage: "Tiller" @@ -397,7 +396,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-announce-2 - namespace: default + namespace: argocd labels: app: redis-ha heritage: "Tiller" @@ -428,7 +427,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha - namespace: default + namespace: argocd labels: app: redis-ha heritage: "Tiller" @@ -457,7 +456,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-haproxy - namespace: default + namespace: argocd labels: app: redis-ha heritage: "Tiller" @@ -483,7 +482,7 @@ kind: Deployment apiVersion: apps/v1 metadata: name: argocd-redis-ha-haproxy - namespace: default + namespace: argocd labels: app: redis-ha heritage: "Tiller" @@ -591,7 +590,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: argocd-redis-ha-server - namespace: default + namespace: argocd labels: argocd-redis-ha: replica app: redis-ha diff --git a/manifests/ha/base/redis-ha/generate.sh b/manifests/ha/base/redis-ha/generate.sh index d93998bced27a..0777b49a23ec0 100755 --- a/manifests/ha/base/redis-ha/generate.sh +++ b/manifests/ha/base/redis-ha/generate.sh @@ -16,4 +16,6 @@ helm2 template ./chart \ --name argocd \ --values ./chart/values.yaml \ ${helm_execute} \ - >> ./chart/upstream.yaml + >> ./chart/upstream_orig.yaml + +sed -e 's/check inter 1s/check inter 3s/' ./chart/upstream_orig.yaml > ./chart/upstream.yaml && rm ./chart/upstream_orig.yaml diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index 6869230edfb25..6b53e680350dd 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -2230,9 +2230,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE0 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_1 mode tcp @@ -2244,9 +2244,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE1 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_2 mode tcp @@ -2258,9 +2258,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE2 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # decide redis backend to use #master @@ -2279,11 +2279,11 @@ data: tcp-check send QUIT\r\n tcp-check expect string +OK use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 } - server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1 + server R0 argocd-redis-ha-announce-0:6379 check inter 3s fall 1 rise 1 use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 } - server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1 + server R1 argocd-redis-ha-announce-1:6379 check inter 3s fall 1 rise 1 use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 } - server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1 + server R2 argocd-redis-ha-announce-2:6379 check inter 3s fall 1 rise 1 haproxy_init.sh: | HAPROXY_CONF=/data/haproxy.cfg cp /readonly/haproxy.cfg "$HAPROXY_CONF" diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml index 5c8641464a009..e3b1afc1c9855 100644 --- a/manifests/ha/namespace-install.yaml +++ b/manifests/ha/namespace-install.yaml @@ -2145,9 +2145,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE0 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_1 mode tcp @@ -2159,9 +2159,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE1 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # Check Sentinel and whether they are nominated master backend check_if_redis_is_master_2 mode tcp @@ -2173,9 +2173,9 @@ data: tcp-check expect string REPLACE_ANNOUNCE2 tcp-check send QUIT\r\n tcp-check expect string +OK - server R0 argocd-redis-ha-announce-0:26379 check inter 1s - server R1 argocd-redis-ha-announce-1:26379 check inter 1s - server R2 argocd-redis-ha-announce-2:26379 check inter 1s + server R0 argocd-redis-ha-announce-0:26379 check inter 3s + server R1 argocd-redis-ha-announce-1:26379 check inter 3s + server R2 argocd-redis-ha-announce-2:26379 check inter 3s # decide redis backend to use #master @@ -2194,11 +2194,11 @@ data: tcp-check send QUIT\r\n tcp-check expect string +OK use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 } - server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1 + server R0 argocd-redis-ha-announce-0:6379 check inter 3s fall 1 rise 1 use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 } - server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1 + server R1 argocd-redis-ha-announce-1:6379 check inter 3s fall 1 rise 1 use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 } - server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1 + server R2 argocd-redis-ha-announce-2:6379 check inter 3s fall 1 rise 1 haproxy_init.sh: | HAPROXY_CONF=/data/haproxy.cfg cp /readonly/haproxy.cfg "$HAPROXY_CONF" From 3009531a8d89734959e2a34a4cf78203af2da114 Mon Sep 17 00:00:00 2001 From: jannfis Date: Fri, 3 Apr 2020 22:13:17 +0200 Subject: [PATCH 2/2] Restore original namespace --- .../ha/base/redis-ha/chart/upstream.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/manifests/ha/base/redis-ha/chart/upstream.yaml b/manifests/ha/base/redis-ha/chart/upstream.yaml index 5641982459e97..a7d37958c42bd 100644 --- a/manifests/ha/base/redis-ha/chart/upstream.yaml +++ b/manifests/ha/base/redis-ha/chart/upstream.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: argocd-redis-ha-configmap - namespace: argocd + namespace: default labels: heritage: Tiller release: argocd @@ -267,7 +267,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: argocd-redis-ha - namespace: argocd + namespace: default labels: heritage: Tiller release: argocd @@ -281,7 +281,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: argocd-redis-ha-haproxy - namespace: argocd + namespace: default labels: heritage: Tiller release: argocd @@ -295,7 +295,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: argocd-redis-ha - namespace: argocd + namespace: default labels: heritage: Tiller release: argocd @@ -316,7 +316,7 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: argocd-redis-ha - namespace: argocd + namespace: default labels: heritage: Tiller release: argocd @@ -338,7 +338,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-announce-0 - namespace: argocd + namespace: default labels: app: redis-ha heritage: "Tiller" @@ -367,7 +367,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-announce-1 - namespace: argocd + namespace: default labels: app: redis-ha heritage: "Tiller" @@ -396,7 +396,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-announce-2 - namespace: argocd + namespace: default labels: app: redis-ha heritage: "Tiller" @@ -427,7 +427,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha - namespace: argocd + namespace: default labels: app: redis-ha heritage: "Tiller" @@ -456,7 +456,7 @@ apiVersion: v1 kind: Service metadata: name: argocd-redis-ha-haproxy - namespace: argocd + namespace: default labels: app: redis-ha heritage: "Tiller" @@ -482,7 +482,7 @@ kind: Deployment apiVersion: apps/v1 metadata: name: argocd-redis-ha-haproxy - namespace: argocd + namespace: default labels: app: redis-ha heritage: "Tiller" @@ -590,7 +590,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: argocd-redis-ha-server - namespace: argocd + namespace: default labels: argocd-redis-ha: replica app: redis-ha