Skip to content

Commit

Permalink
fix(health): improved healthcheck worker
Browse files Browse the repository at this point in the history
  • Loading branch information
cnlangzi committed Jun 8, 2022
1 parent 190abf1 commit fc3a43d
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 19 deletions.
2 changes: 2 additions & 0 deletions code/go/0chain.net/blobber/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ func setupConfig(configDir string, deploymentMode int) {
config.Configuration.RMRedeemFreq = viper.GetInt64("readmarker_redeem.frequency")
config.Configuration.RMRedeemNumWorkers = viper.GetInt("readmarker_redeem.num_workers")

config.Configuration.HealthCheckWorkerFreq = viper.GetDuration("healthcheck.frequency")

config.Configuration.ChallengeResolveFreq = viper.GetInt64("challenge_response.frequency")
config.Configuration.ChallengeResolveNumWorkers = viper.GetInt("challenge_response.num_workers")
config.Configuration.ChallengeMaxRetires = viper.GetInt("challenge_response.max_retries")
Expand Down
20 changes: 12 additions & 8 deletions code/go/0chain.net/blobber/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,23 @@ func refreshPriceOnChain(ctx context.Context) {
func startHealthCheck(ctx context.Context) {

const REPEAT_DELAY = 10 * 1 // 60s
var err error

for {
select {
case <-ctx.Done():
break
case <-time.After(REPEAT_DELAY * time.Second):

err = handler.SendHealthCheck()
if err == nil {
logging.Logger.Info("success to send heartbeat")
} else {
logging.Logger.Warn("failed to send heartbeat", zap.Error(err))
}
go func() {
start := time.Now()

txnHash, err := handler.SendHealthCheck()
end := time.Now()
if err == nil {
logging.Logger.Info("success to send heartbeat", zap.String("txn_hash", txnHash), zap.Time("start", start), zap.Time("end", end), zap.Duration("duration", end.Sub(start)))
} else {
logging.Logger.Warn("failed to send heartbeat", zap.String("txn_hash", txnHash), zap.Time("start", start), zap.Time("end", end), zap.Duration("duration", end.Sub(start)))
}
}()
}
}
}
Expand Down
4 changes: 4 additions & 0 deletions code/go/0chain.net/blobbercore/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ func SetupDefaultConfig() {
viper.SetDefault("challenge_response.num_workers", 5)
viper.SetDefault("challenge_response.max_retries", 10)

viper.SetDefault("healthcheck.frequency", "60s")

viper.SetDefault("capacity", -1)
viper.SetDefault("read_price", 0.0)
viper.SetDefault("write_price", 0.0)
Expand Down Expand Up @@ -107,6 +109,8 @@ type Config struct {
ColdStorageDeleteLocalCopy bool
ColdStorageDeleteCloudCopy bool

HealthCheckWorkerFreq time.Duration

MinioStart bool
MinioWorkerFreq int64
MinioUseSSL bool
Expand Down
19 changes: 9 additions & 10 deletions code/go/0chain.net/blobbercore/handler/protocol.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package handler
import (
"context"
"errors"
"log"
"sync"
"time"

Expand Down Expand Up @@ -104,8 +103,13 @@ func RegisterBlobber(ctx context.Context) error {
return nil
}

return SendHealthCheck()
txnHash, err := SendHealthCheck()
if err != nil {
logging.Logger.Error("Failed to send healthcheck transaction", zap.String("txn_hash", txnHash))
return err
}

return nil
}

// UpdateBlobber update blobber
Expand Down Expand Up @@ -239,17 +243,12 @@ func WalletRegister() error {
}

// SendHealthCheck send heartbeat to blockchain
func SendHealthCheck() error {
log.Println("SendHealthCheck")
func SendHealthCheck() (string, error) {
txnHash, err := BlobberHealthCheck()
if err != nil {
return err
return txnHash, err
}
_, err = TransactionVerify(txnHash)
if err != nil {
logging.Logger.Error("Failed to verify blobber health check", zap.Any("err", err), zap.String("txn.Hash", txnHash))
return err
}

return nil
return txnHash, err
}
6 changes: 5 additions & 1 deletion config/0chain_blobber.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: "1.0"

logging:
level: "error"
level: "info"
console: true # printing log to console is only supported in development mode

info:
Expand Down Expand Up @@ -93,6 +93,10 @@ challenge_response:
frequency: 10
num_workers: 5
max_retries: 20

healthcheck:
frequency: 60s # send healthcheck to miners every 60 seconds

pg:
user: postgres
password: postgres
Expand Down

0 comments on commit fc3a43d

Please sign in to comment.