(Sentinel) DAL rest network delay checker (#2326)

* feat: rest network delay checker

* feat: define dal check interval

* fix: update based on feedback

node/pkg/checker/dal/app.go

@@ -109,11 +109,12 @@ func Start(ctx context.Context) error {
 	}
 	defer pool.Close()
 
+	networkDelayAlarmCount := 0
 	alarmCount := map[string]int{}
 	wsDelayAlarmCount := map[string]int{}
 
 	for range ticker.C {
-		err := checkDal(endpoint, key, alarmCount)
+		err := checkDal(endpoint, key, alarmCount, &networkDelayAlarmCount)
 		if err != nil {
 			log.Error().Str("Player", "DalChecker").Err(err).Msg("error in checkDal")
 		}
@@ -136,7 +137,7 @@ func buildSubscriptionParams(configs []Config) []string {
 	return params
 }
 
-func checkDal(endpoint string, key string, alarmCount map[string]int) error {
+func checkDal(endpoint string, key string, alarmCount map[string]int, networkDelayAlarmCount *int) error {
 	msg := ""
 
 	now := time.Now()
@@ -147,6 +148,16 @@ func checkDal(endpoint string, key string, alarmCount map[string]int) error {
 	)
 	networkDelay := time.Since(now)
 
+	if networkDelay > NetworkDelayThreshold {
+		*networkDelayAlarmCount++
+		if *networkDelayAlarmCount > AlarmOffsetInTotal {
+			msg += fmt.Sprintf("(DAL) network delay: %s\n", networkDelay)
+			*networkDelayAlarmCount = 0
+		}
+	} else {
+		*networkDelayAlarmCount = 0
+	}
+
 	if err != nil {
 		return err
 	}

node/pkg/checker/dal/types.go

@@ -12,6 +12,7 @@ const (
 	AlarmOffsetInTotal      = 3
 	WsDelayThreshold        = 9 * time.Second
 	WsPushThreshold         = 5 * time.Second
+	NetworkDelayThreshold   = 6 * time.Second
 	IgnoreKeys              = "test,sentinel,orakl_reporter"
 
 	TrafficCheckQuery = `select count(1) from rest_calls where