Skip to content

Commit

Permalink
fix: Increase HAProxy check interval to prevent intermittent failures (…
Browse files Browse the repository at this point in the history 
…#3356)

* Increase HAProxy check interval to prevent intermittent failures/state flapping

* Restore original namespace
  • Loading branch information
@jannfis @alexmt
jannfis authored and alexmt committed Apr 6, 2020
1 parent 76241f9 commit a069639
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 38 deletions.
25 changes: 12 additions & 13 deletions manifests/ha/base/redis-ha/chart/upstream.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# This is an auto-generated file. DO NOT EDIT
---
# Source: redis-ha/charts/redis-ha/templates/redis-ha-configmap.yaml
apiVersion: v1
Expand Down Expand Up @@ -155,9 +154,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE0
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_1
mode tcp
Expand All @@ -169,9 +168,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE1
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_2
mode tcp
Expand All @@ -183,9 +182,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE2
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# decide redis backend to use
#master
Expand All @@ -204,11 +203,11 @@ data:
tcp-check send QUIT\r\n
tcp-check expect string +OK
use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 }
server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1
server R0 argocd-redis-ha-announce-0:6379 check inter 3s fall 1 rise 1
use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 }
server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1
server R1 argocd-redis-ha-announce-1:6379 check inter 3s fall 1 rise 1
use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 }
server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1
server R2 argocd-redis-ha-announce-2:6379 check inter 3s fall 1 rise 1
haproxy_init.sh: |
HAPROXY_CONF=/data/haproxy.cfg
cp /readonly/haproxy.cfg "$HAPROXY_CONF"
Expand Down
4 changes: 3 additions & 1 deletion manifests/ha/base/redis-ha/generate.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,6 @@ helm2 template ./chart \
--name argocd \
--values ./chart/values.yaml \
${helm_execute} \
>> ./chart/upstream.yaml
>> ./chart/upstream_orig.yaml

sed -e 's/check inter 1s/check inter 3s/' ./chart/upstream_orig.yaml > ./chart/upstream.yaml && rm ./chart/upstream_orig.yaml
24 changes: 12 additions & 12 deletions manifests/ha/install.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2213,9 +2213,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE0
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_1
mode tcp
Expand All @@ -2227,9 +2227,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE1
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_2
mode tcp
Expand All @@ -2241,9 +2241,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE2
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# decide redis backend to use
#master
Expand All @@ -2262,11 +2262,11 @@ data:
tcp-check send QUIT\r\n
tcp-check expect string +OK
use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 }
server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1
server R0 argocd-redis-ha-announce-0:6379 check inter 3s fall 1 rise 1
use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 }
server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1
server R1 argocd-redis-ha-announce-1:6379 check inter 3s fall 1 rise 1
use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 }
server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1
server R2 argocd-redis-ha-announce-2:6379 check inter 3s fall 1 rise 1
haproxy_init.sh: |
HAPROXY_CONF=/data/haproxy.cfg
cp /readonly/haproxy.cfg "$HAPROXY_CONF"
Expand Down
24 changes: 12 additions & 12 deletions manifests/ha/namespace-install.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2128,9 +2128,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE0
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_1
mode tcp
Expand All @@ -2142,9 +2142,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE1
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# Check Sentinel and whether they are nominated master
backend check_if_redis_is_master_2
mode tcp
Expand All @@ -2156,9 +2156,9 @@ data:
tcp-check expect string REPLACE_ANNOUNCE2
tcp-check send QUIT\r\n
tcp-check expect string +OK
server R0 argocd-redis-ha-announce-0:26379 check inter 1s
server R1 argocd-redis-ha-announce-1:26379 check inter 1s
server R2 argocd-redis-ha-announce-2:26379 check inter 1s
server R0 argocd-redis-ha-announce-0:26379 check inter 3s
server R1 argocd-redis-ha-announce-1:26379 check inter 3s
server R2 argocd-redis-ha-announce-2:26379 check inter 3s
# decide redis backend to use
#master
Expand All @@ -2177,11 +2177,11 @@ data:
tcp-check send QUIT\r\n
tcp-check expect string +OK
use-server R0 if { srv_is_up(R0) } { nbsrv(check_if_redis_is_master_0) ge 2 }
server R0 argocd-redis-ha-announce-0:6379 check inter 1s fall 1 rise 1
server R0 argocd-redis-ha-announce-0:6379 check inter 3s fall 1 rise 1
use-server R1 if { srv_is_up(R1) } { nbsrv(check_if_redis_is_master_1) ge 2 }
server R1 argocd-redis-ha-announce-1:6379 check inter 1s fall 1 rise 1
server R1 argocd-redis-ha-announce-1:6379 check inter 3s fall 1 rise 1
use-server R2 if { srv_is_up(R2) } { nbsrv(check_if_redis_is_master_2) ge 2 }
server R2 argocd-redis-ha-announce-2:6379 check inter 1s fall 1 rise 1
server R2 argocd-redis-ha-announce-2:6379 check inter 3s fall 1 rise 1
haproxy_init.sh: |
HAPROXY_CONF=/data/haproxy.cfg
cp /readonly/haproxy.cfg "$HAPROXY_CONF"
Expand Down

0 comments on commit a069639

Please sign in to comment.