From 901c27f31a861a2da8e67668d648316086c087a6 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 12 Aug 2024 19:35:07 +0200 Subject: [PATCH 01/48] feat: setup circuit breaker --- go.mod | 1 + go.sum | 2 + internal/pkg/circuitbreaker/circuitbreaker.go | 78 +++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 internal/pkg/circuitbreaker/circuitbreaker.go diff --git a/go.mod b/go.mod index 75e9afb1ef..febf6cfe93 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/segmentio/kafka-go v0.4.44 github.com/sirupsen/logrus v1.9.3 github.com/slack-go/slack v0.10.2 + github.com/sony/gobreaker/v2 v2.0.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index cfeca4f84d..98f7864557 100644 --- a/go.sum +++ b/go.sum @@ -1792,6 +1792,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sony/gobreaker/v2 v2.0.0 h1:23AaR4JQ65y4rz8JWMzgXw2gKOykZ/qfqYunll4OwJ4= +github.com/sony/gobreaker/v2 v2.0.0/go.mod h1:8JnRUz80DJ1/ne8M8v7nmTs2713i58nIt4s7XcGe/DI= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY= github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI= diff --git a/internal/pkg/circuitbreaker/circuitbreaker.go b/internal/pkg/circuitbreaker/circuitbreaker.go new file mode 100644 index 0000000000..fc266ad295 --- /dev/null +++ b/internal/pkg/circuitbreaker/circuitbreaker.go @@ -0,0 +1,78 @@ +package circuitbreaker + +import ( + "context" + "fmt" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/pkg/rdb" + "github.com/redis/go-redis/v9" + "github.com/sony/gobreaker/v2" + "time" +) + +type CircuitBreaker interface { + Execute() error +} + +func NewCircuitBreaker(name string, cfg config.RedisConfiguration) (CircuitBreaker, error) { + r, err := rdb.NewClient(cfg.BuildDsn()) + if err != nil { + return nil, err + } + return NewRedisCircuitBreaker(name, r.Client()) +} + +// State is a type that represents a state of the CircuitBreaker. +type State int + +const prefix = "breaker" + +type RedisCircuitBreaker struct { + breaker *gobreaker.CircuitBreaker[bool] + resetDuration time.Duration +} + +type BreakerEntry struct { + Requests uint32 `json:"requests"` + TotalFailures uint32 `json:"total_failures"` + TotalSuccesses uint32 `json:"total_successes"` + LastTriggeredAt time.Time `json:"last_triggered_at"` + ConsecutiveFailures uint32 `json:"consecutive_failures"` + ConsecutiveSuccesses uint32 `json:"consecutive_successes"` +} + +func NewRedisCircuitBreaker(name string, client redis.UniversalClient) (*RedisCircuitBreaker, error) { + // build the redis key + key := fmt.Sprintf("%s:%s", prefix, name) + + // load breaker state from redis into memory + val := client.Get(context.Background(), key).Val() + if val != redis.Nil.Error() { + // init breaker here + } + + // if state is nil create a new breaker + b := gobreaker.NewCircuitBreaker[bool](gobreaker.Settings{ + Name: name, + MaxRequests: 0, + Interval: time.Second * 30, + Timeout: time.Second * 5, + ReadyToTrip: func(counts gobreaker.Counts) bool { + return counts.ConsecutiveFailures > 0 + }, + }) + + return &RedisCircuitBreaker{ + breaker: b, + }, nil +} + +func (c *RedisCircuitBreaker) Execute() error { + // run the circuit breaker's Execute func + + // get the state from redis again + + // update the state on redis + + return nil +} From fc355f90e92b01f8c42ced3a8e8d94c578248667 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 16 Aug 2024 17:33:03 +0200 Subject: [PATCH 02/48] feat: add circuit breaker switching implementation --- cmd/worker/worker.go | 5 + internal/pkg/circuitbreaker/circuitbreaker.go | 78 ----- pkg/circuit_breaker/circuit_breaker.go | 296 ++++++++++++++++++ pkg/circuit_breaker/circuit_breaker_test.go | 88 ++++++ pkg/clock/clock.go | 54 ++++ pkg/clock/clock_test.go | 44 +++ 6 files changed, 487 insertions(+), 78 deletions(-) delete mode 100644 internal/pkg/circuitbreaker/circuitbreaker.go create mode 100644 pkg/circuit_breaker/circuit_breaker.go create mode 100644 pkg/circuit_breaker/circuit_breaker_test.go create mode 100644 pkg/clock/clock.go create mode 100644 pkg/clock/clock_test.go diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index f7b768a10c..098bf081e6 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -16,6 +16,8 @@ import ( "github.com/frain-dev/convoy/internal/pkg/smtp" "github.com/frain-dev/convoy/internal/telemetry" "github.com/frain-dev/convoy/net" + "github.com/frain-dev/convoy/pkg/circuit_breaker" + "github.com/frain-dev/convoy/pkg/clock" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/queue" redisQueue "github.com/frain-dev/convoy/queue/redis" @@ -242,6 +244,9 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } + breaker := circuit_breaker.NewCircuitBreakerManager(rd.Client(), a.DB.GetDB(), clock.NewRealClock()) + go breaker.Run(ctx) + consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, eventDeliveryRepo, diff --git a/internal/pkg/circuitbreaker/circuitbreaker.go b/internal/pkg/circuitbreaker/circuitbreaker.go deleted file mode 100644 index fc266ad295..0000000000 --- a/internal/pkg/circuitbreaker/circuitbreaker.go +++ /dev/null @@ -1,78 +0,0 @@ -package circuitbreaker - -import ( - "context" - "fmt" - "github.com/frain-dev/convoy/config" - "github.com/frain-dev/convoy/internal/pkg/rdb" - "github.com/redis/go-redis/v9" - "github.com/sony/gobreaker/v2" - "time" -) - -type CircuitBreaker interface { - Execute() error -} - -func NewCircuitBreaker(name string, cfg config.RedisConfiguration) (CircuitBreaker, error) { - r, err := rdb.NewClient(cfg.BuildDsn()) - if err != nil { - return nil, err - } - return NewRedisCircuitBreaker(name, r.Client()) -} - -// State is a type that represents a state of the CircuitBreaker. -type State int - -const prefix = "breaker" - -type RedisCircuitBreaker struct { - breaker *gobreaker.CircuitBreaker[bool] - resetDuration time.Duration -} - -type BreakerEntry struct { - Requests uint32 `json:"requests"` - TotalFailures uint32 `json:"total_failures"` - TotalSuccesses uint32 `json:"total_successes"` - LastTriggeredAt time.Time `json:"last_triggered_at"` - ConsecutiveFailures uint32 `json:"consecutive_failures"` - ConsecutiveSuccesses uint32 `json:"consecutive_successes"` -} - -func NewRedisCircuitBreaker(name string, client redis.UniversalClient) (*RedisCircuitBreaker, error) { - // build the redis key - key := fmt.Sprintf("%s:%s", prefix, name) - - // load breaker state from redis into memory - val := client.Get(context.Background(), key).Val() - if val != redis.Nil.Error() { - // init breaker here - } - - // if state is nil create a new breaker - b := gobreaker.NewCircuitBreaker[bool](gobreaker.Settings{ - Name: name, - MaxRequests: 0, - Interval: time.Second * 30, - Timeout: time.Second * 5, - ReadyToTrip: func(counts gobreaker.Counts) bool { - return counts.ConsecutiveFailures > 0 - }, - }) - - return &RedisCircuitBreaker{ - breaker: b, - }, nil -} - -func (c *RedisCircuitBreaker) Execute() error { - // run the circuit breaker's Execute func - - // get the state from redis again - - // update the state on redis - - return nil -} diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go new file mode 100644 index 0000000000..2ab118b3a8 --- /dev/null +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -0,0 +1,296 @@ +package circuit_breaker + +import ( + "context" + "errors" + "fmt" + "github.com/frain-dev/convoy/pkg/clock" + "github.com/frain-dev/convoy/pkg/log" + "github.com/frain-dev/convoy/pkg/msgpack" + "github.com/jmoiron/sqlx" + "github.com/redis/go-redis/v9" + "time" +) + +const prefix = "breaker" + +var ( + // ErrTooManyRequests is returned when the CB state is half open and the requests count is over the cb maxRequests + ErrTooManyRequests = errors.New("too many requests") + // ErrOpenState is returned when the CB state is open + ErrOpenState = errors.New("circuit breaker is open") +) + +// State is a type that represents a state of CircuitBreaker. +type State int + +func stateFromString(s string) State { + switch s { + case "open": + return StateOpen + case "closed": + return StateClosed + case "half-open": + return StateHalfOpen + } + return StateClosed +} + +// These constants are states of the CircuitBreaker. +const ( + StateClosed State = iota + StateHalfOpen + StateOpen +) + +func (s State) String() string { + switch s { + case StateClosed: + return "closed" + case StateHalfOpen: + return "half-open" + case StateOpen: + return "open" + default: + return fmt.Sprintf("unknown state: %d", s) + } +} + +type CircuitBreaker struct { + State State `json:"state"` + Requests float64 `json:"requests"` + EndpointID string `json:"endpoint_id"` + TotalFailures float64 `json:"total_failures"` + TotalSuccesses float64 `json:"total_successes"` + WillResetAt time.Time `json:"will_reset_at"` + ConsecutiveFailures float64 `json:"consecutive_failures"` + ConsecutiveSuccesses float64 `json:"consecutive_successes"` +} + +func (b *CircuitBreaker) String() (s string, err error) { + bytes, err := msgpack.EncodeMsgPack(b) + if err != nil { + return "", err + } + + return string(bytes), nil +} + +func (b *CircuitBreaker) tripCircuitBreaker(resetTime time.Time) { + b.State = StateOpen + b.WillResetAt = resetTime + b.ConsecutiveFailures++ +} + +func (b *CircuitBreaker) toHalfOpen() { + b.State = StateHalfOpen +} + +func (b *CircuitBreaker) resetCircuitBreaker() { + b.State = StateClosed + b.ConsecutiveFailures = 0 +} + +type DBPollResult struct { + EndpointID string `json:"endpoint_id" db:"endpoint_id"` + Failures float64 `json:"failures" db:"failures"` + Successes float64 `json:"successes" db:"successes"` + FailRate float64 `json:"fail_rate"` +} + +func (pr *DBPollResult) CalculateFailRate() { + pr.FailRate = 1 + (pr.Failures / (pr.Successes + pr.Failures)) +} + +type CircuitBreakerManager struct { + breakers []CircuitBreaker + clock clock.Clock + redis *redis.Client + db *sqlx.DB +} + +func NewCircuitBreakerManager(client redis.UniversalClient, db *sqlx.DB, clock clock.Clock) *CircuitBreakerManager { + // todo(raymond): define and load breaker config + r := &CircuitBreakerManager{ + db: db, + clock: clock, + redis: client.(*redis.Client), + } + + return r +} + +func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, dbPollResults []DBPollResult) error { + for i := range dbPollResults { + dbPollResults[i].FailRate = dbPollResults[i].Failures / (dbPollResults[i].Successes + dbPollResults[i].Failures) + } + + var keys []string + for i := range dbPollResults { + key := fmt.Sprintf("%s:%s", prefix, dbPollResults[i].EndpointID) + keys = append(keys, key) + dbPollResults[i].EndpointID = key + } + + res, err := cb.redis.MGet(context.Background(), keys...).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return err + } + + var circuitBreakers []CircuitBreaker + for i := range res { + if res[i] == nil { + c := CircuitBreaker{ + State: StateClosed, + EndpointID: dbPollResults[i].EndpointID, + WillResetAt: cb.clock.Now().Add(time.Second * 30), + } + circuitBreakers = append(circuitBreakers, c) + continue + } + + c := CircuitBreaker{} + asBytes := []byte(res[i].(string)) + innerErr := msgpack.DecodeMsgPack(asBytes, &c) + if innerErr != nil { + return innerErr + } + + circuitBreakers = append(circuitBreakers, c) + } + + resultsMap := make(map[string]DBPollResult) + for _, result := range dbPollResults { + resultsMap[result.EndpointID] = result + } + + circuitBreakerMap := make(map[string]CircuitBreaker, len(resultsMap)) + + for _, breaker := range circuitBreakers { + // todo(raymond): these should be part of the config + // 10% of total failed requests in the observability window + threshold := 0.1 + // total number of failed requests in the observability window + failureCount := 1.0 + + result := resultsMap[breaker.EndpointID] + fmt.Printf("result: %+v\n", result) + + // Apply the logic to decide whether to trip the breaker + if result.FailRate > threshold || result.Failures >= failureCount { + breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Second * 30)) + } else if breaker.State == StateHalfOpen && result.Successes > 0 { + breaker.resetCircuitBreaker() + } else if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { + breaker.toHalfOpen() + } + + breaker.Requests = result.Successes + result.Failures + breaker.TotalFailures = result.Failures + breaker.TotalSuccesses = result.Successes + + circuitBreakerMap[breaker.EndpointID] = breaker + } + + // Update the circuit breaker state in Redis + if err = cb.updateCircuitBreakersInRedis(ctx, circuitBreakerMap); err != nil { + log.WithError(err).Error("Failed to update state in Redis") + } + + return nil +} + +// todo(raymond): move this to the delivery attempts repo +func (cb *CircuitBreakerManager) getFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []DBPollResult, err error) { + query := ` + SELECT + endpoint_id, + COUNT(CASE WHEN status = false THEN 1 END) AS failures, + COUNT(CASE WHEN status = true THEN 1 END) AS successes + FROM convoy.delivery_attempts + WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) group by endpoint_id; + ` + + rows, err := cb.db.QueryxContext(ctx, query, lookBackDuration) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var rowValue DBPollResult + if rowScanErr := rows.StructScan(&rowValue); rowScanErr != nil { + return nil, rowScanErr + } + results = append(results, rowValue) + } + + return results, nil +} + +func (cb *CircuitBreakerManager) updateCircuitBreakersInRedis(ctx context.Context, breakers map[string]CircuitBreaker) error { + breakerStringsMap := make(map[string]string, len(breakers)) + for key, breaker := range breakers { + val, err := breaker.String() + if err != nil { + return err + } + breakerStringsMap[key] = val + } + + // Update the state + err := cb.redis.MSet(ctx, breakerStringsMap).Err() + if err != nil { + return err + } + + return nil +} + +func (cb *CircuitBreakerManager) loadCircuitBreakerStateFromRedis(ctx context.Context) ([]CircuitBreaker, error) { + keys, err := cb.redis.Keys(ctx, "breaker*").Result() + if err != nil { + return nil, err + } + + res, err := cb.redis.MGet(ctx, keys...).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return nil, err + } + + var circuitBreakers []CircuitBreaker + for i := range res { + c := CircuitBreaker{} + asBytes := []byte(res[i].(string)) + innerErr := msgpack.DecodeMsgPack(asBytes, &c) + if innerErr != nil { + return nil, innerErr + } + + circuitBreakers = append(circuitBreakers, c) + } + + return circuitBreakers, nil +} + +func (cb *CircuitBreakerManager) Run(ctx context.Context) { + lookBackDuration := 5 + for { + // Get the failure and success counts from the last X minutes + dbPollResults, err := cb.getFailureAndSuccessCounts(ctx, lookBackDuration) + if err != nil { + log.WithError(err).Error("poll db failed") + } + + if len(dbPollResults) == 0 { + // there's nothing to update + continue + } + + err = cb.sampleEventsAndUpdateState(ctx, dbPollResults) + if err != nil { + log.WithError(err).Error("Failed to sample events and update state") + } + time.Sleep(30 * time.Second) + } +} diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go new file mode 100644 index 0000000000..b2c2a7f12c --- /dev/null +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -0,0 +1,88 @@ +package circuit_breaker + +import ( + "context" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/database/postgres" + "github.com/frain-dev/convoy/internal/pkg/rdb" + "github.com/frain-dev/convoy/pkg/clock" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestNewCircuitBreaker(t *testing.T) { + ctx := context.Background() + + re, err := rdb.NewClient([]string{"redis://localhost:6379"}) + require.NoError(t, err) + + keys, err := re.Client().Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + err = re.Client().Del(ctx, keys...).Err() + require.NoError(t, err) + + db, err := postgres.NewDB(config.Configuration{ + Database: config.DatabaseConfiguration{ + Type: config.PostgresDatabaseProvider, + Scheme: "postgres", + Host: "localhost", + Username: "postgres", + Password: "postgres", + Database: "endpoint_fix", + Options: "sslmode=disable&connect_timeout=30", + Port: 5432, + }, + }) + require.NoError(t, err) + + testClock := clock.NewSimulatedClock(time.Now()) + + b := NewCircuitBreakerManager(re.Client(), db.GetDB(), testClock) + + endpointId := "endpoint-1" + pollResults := [][]DBPollResult{ + { + DBPollResult{ + EndpointID: endpointId, + Failures: 1, + Successes: 0, + }, + }, + { + DBPollResult{ + EndpointID: endpointId, + Failures: 1, + Successes: 0, + }, + }, + { + DBPollResult{ + EndpointID: endpointId, + Failures: 0, + Successes: 0, + }, + }, + { + DBPollResult{ + EndpointID: endpointId, + Failures: 0, + Successes: 1, + }, + }, + } + + for i := 0; i < len(pollResults); i++ { + innerErr := b.sampleEventsAndUpdateState(ctx, pollResults[i]) + require.NoError(t, innerErr) + + breakers, innerErr := b.loadCircuitBreakerStateFromRedis(ctx) + require.NoError(t, innerErr) + + require.Equal(t, len(breakers), 1) + t.Log(breakers) + + testClock.AdvanceTime(time.Minute) + } +} diff --git a/pkg/clock/clock.go b/pkg/clock/clock.go new file mode 100644 index 0000000000..c2a79dd97f --- /dev/null +++ b/pkg/clock/clock.go @@ -0,0 +1,54 @@ +package clock + +import ( + "sync" + "time" +) + +// A Clock is an object that can tell you the current time. +// +// This interface allows decoupling code that uses time from the code that creates +// a point in time. You can use this to your advantage by injecting Clocks into interfaces +// rather than having implementations call time.Now() directly. +// +// Use RealClock() in production. +// Use SimulatedClock() in test. +type Clock interface { + Now() time.Time +} + +func NewRealClock() Clock { return &realTimeClock{} } + +type realTimeClock struct{} + +func (_ *realTimeClock) Now() time.Time { return time.Now() } + +// A SimulatedClock is a concrete Clock implementation that doesn't "tick" on its own. +// Time is advanced by explicit call to the AdvanceTime() or SetTime() functions. +// This object is concurrency safe. +type SimulatedClock struct { + mu sync.Mutex + t time.Time // guarded by mu +} + +func NewSimulatedClock(t time.Time) *SimulatedClock { + return &SimulatedClock{t: t} +} + +func (c *SimulatedClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *SimulatedClock) SetTime(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = t +} + +func (c *SimulatedClock) AdvanceTime(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} diff --git a/pkg/clock/clock_test.go b/pkg/clock/clock_test.go new file mode 100644 index 0000000000..3b6f381e4b --- /dev/null +++ b/pkg/clock/clock_test.go @@ -0,0 +1,44 @@ +package clock + +import ( + "testing" + "time" +) + +func TestSimulatedClock(t *testing.T) { + now := time.Now() + + tests := []struct { + desc string + initTime time.Time + advanceBy time.Duration + wantTime time.Time + }{ + { + desc: "advance time forward", + initTime: now, + advanceBy: 30 * time.Second, + wantTime: now.Add(30 * time.Second), + }, + { + desc: "advance time backward", + initTime: now, + advanceBy: -10 * time.Second, + wantTime: now.Add(-10 * time.Second), + }, + } + + for _, tc := range tests { + c := NewSimulatedClock(tc.initTime) + + if c.Now() != tc.initTime { + t.Errorf("%s: Before Advance; SimulatedClock.Now() = %v, want %v", tc.desc, c.Now(), tc.initTime) + } + + c.AdvanceTime(tc.advanceBy) + + if c.Now() != tc.wantTime { + t.Errorf("%s: After Advance; SimulatedClock.Now() = %v, want %v", tc.desc, c.Now(), tc.wantTime) + } + } +} From 6f691e82cd1475a95b84dc17d82f9f6e487a7e8e Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 19 Aug 2024 16:29:26 +0200 Subject: [PATCH 03/48] feat: add circuit breaker configuration --- pkg/circuit_breaker/circuit_breaker.go | 122 +++++++++++++------- pkg/circuit_breaker/circuit_breaker_test.go | 25 ++-- 2 files changed, 96 insertions(+), 51 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 2ab118b3a8..8882b41cfb 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -21,6 +21,50 @@ var ( ErrOpenState = errors.New("circuit breaker is open") ) +// CircuitBreakerConfig is config which all the circuit breakers that manager manages will use +// +// { +// "sample_time": 5, +// "duration": 5, +// "error_timeout": 50, +// "error_threshold": 70, +// "failure_count": 10, +// "success_threshold": 10, +// "consecutive_failure_threshold": 10, +// "notification_thresholds": [30, 65] +// } +type CircuitBreakerConfig struct { + // SampleTime is the time interval (in seconds) at which the data source + // is polled to determine the number successful and failed requests + SampleTime int `json:"sample_time"` + + // ErrorTimeout is the time (in seconds) after which a circuit breaker goes + // into the half-open state from the open state + ErrorTimeout int `json:"error_timeout"` + + // FailureThreshold is the % of failed requests in the observability window + // after which the breaker will go into the open state + FailureThreshold float64 `json:"failure_threshold"` + + // FailureCount total number of failed requests in the observability window + FailureCount int `json:"failure_count"` + + // SuccessThreshold is the % of successful requests in the observability window + // after which a circuit breaker in the half-open state will go into the closed state + SuccessThreshold int `json:"success_threshold"` + + // ObservabilityWindow is how far back in time (in seconds) the data source is + // polled when determining the number successful and failed requests + ObservabilityWindow int `json:"observability_window"` + + // NotificationThresholds These are the error counts after which we will send out notifications. + NotificationThresholds []int `json:"notification_thresholds"` + + // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. + // E.g., after 10 consecutive transitions from half-open → open we should disable it. + ConsecutiveFailureThreshold int `json:"consecutive_failure_threshold"` +} + // State is a type that represents a state of CircuitBreaker. type State int @@ -56,15 +100,18 @@ func (s State) String() string { } } +// CircuitBreaker represents a circuit breaker +// todo(raymond): implement methods to check the state and find out if an action can be performed. type CircuitBreaker struct { State State `json:"state"` - Requests float64 `json:"requests"` + Requests int `json:"requests"` EndpointID string `json:"endpoint_id"` - TotalFailures float64 `json:"total_failures"` - TotalSuccesses float64 `json:"total_successes"` WillResetAt time.Time `json:"will_reset_at"` - ConsecutiveFailures float64 `json:"consecutive_failures"` - ConsecutiveSuccesses float64 `json:"consecutive_successes"` + TotalFailures int `json:"total_failures"` + TotalSuccesses int `json:"total_successes"` + ConsecutiveFailures int `json:"consecutive_failures"` + ConsecutiveSuccesses int `json:"consecutive_successes"` + FailureRate float64 `json:"failure_rate"` } func (b *CircuitBreaker) String() (s string, err error) { @@ -89,42 +136,35 @@ func (b *CircuitBreaker) toHalfOpen() { func (b *CircuitBreaker) resetCircuitBreaker() { b.State = StateClosed b.ConsecutiveFailures = 0 + b.ConsecutiveSuccesses++ } type DBPollResult struct { - EndpointID string `json:"endpoint_id" db:"endpoint_id"` - Failures float64 `json:"failures" db:"failures"` - Successes float64 `json:"successes" db:"successes"` - FailRate float64 `json:"fail_rate"` -} - -func (pr *DBPollResult) CalculateFailRate() { - pr.FailRate = 1 + (pr.Failures / (pr.Successes + pr.Failures)) + EndpointID string `json:"endpoint_id" db:"endpoint_id"` + Failures int `json:"failures" db:"failures"` + Successes int `json:"successes" db:"successes"` } type CircuitBreakerManager struct { breakers []CircuitBreaker + config CircuitBreakerConfig clock clock.Clock redis *redis.Client db *sqlx.DB } -func NewCircuitBreakerManager(client redis.UniversalClient, db *sqlx.DB, clock clock.Clock) *CircuitBreakerManager { - // todo(raymond): define and load breaker config +func NewCircuitBreakerManager(client redis.UniversalClient, db *sqlx.DB, clock clock.Clock, config CircuitBreakerConfig) *CircuitBreakerManager { r := &CircuitBreakerManager{ - db: db, - clock: clock, - redis: client.(*redis.Client), + db: db, + clock: clock, + config: config, + redis: client.(*redis.Client), } return r } func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, dbPollResults []DBPollResult) error { - for i := range dbPollResults { - dbPollResults[i].FailRate = dbPollResults[i].Failures / (dbPollResults[i].Successes + dbPollResults[i].Failures) - } - var keys []string for i := range dbPollResults { key := fmt.Sprintf("%s:%s", prefix, dbPollResults[i].EndpointID) @@ -141,9 +181,8 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, for i := range res { if res[i] == nil { c := CircuitBreaker{ - State: StateClosed, - EndpointID: dbPollResults[i].EndpointID, - WillResetAt: cb.clock.Now().Add(time.Second * 30), + State: StateClosed, + EndpointID: dbPollResults[i].EndpointID, } circuitBreakers = append(circuitBreakers, c) continue @@ -167,27 +206,23 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, circuitBreakerMap := make(map[string]CircuitBreaker, len(resultsMap)) for _, breaker := range circuitBreakers { - // todo(raymond): these should be part of the config - // 10% of total failed requests in the observability window - threshold := 0.1 - // total number of failed requests in the observability window - failureCount := 1.0 - result := resultsMap[breaker.EndpointID] - fmt.Printf("result: %+v\n", result) - - // Apply the logic to decide whether to trip the breaker - if result.FailRate > threshold || result.Failures >= failureCount { - breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Second * 30)) - } else if breaker.State == StateHalfOpen && result.Successes > 0 { - breaker.resetCircuitBreaker() - } else if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { - breaker.toHalfOpen() - } breaker.Requests = result.Successes + result.Failures breaker.TotalFailures = result.Failures breaker.TotalSuccesses = result.Successes + breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.TotalSuccesses+breaker.TotalFailures) + + // todo(raymond): move this to a different place that runs in a goroutine + if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { + breaker.toHalfOpen() + } + + if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { + breaker.resetCircuitBreaker() + } else if breaker.State == StateClosed && (breaker.FailureRate >= cb.config.FailureThreshold || breaker.TotalFailures >= cb.config.FailureCount) { + breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.ErrorTimeout) * time.Second)) + } circuitBreakerMap[breaker.EndpointID] = breaker } @@ -274,10 +309,9 @@ func (cb *CircuitBreakerManager) loadCircuitBreakerStateFromRedis(ctx context.Co } func (cb *CircuitBreakerManager) Run(ctx context.Context) { - lookBackDuration := 5 for { // Get the failure and success counts from the last X minutes - dbPollResults, err := cb.getFailureAndSuccessCounts(ctx, lookBackDuration) + dbPollResults, err := cb.getFailureAndSuccessCounts(ctx, cb.config.ObservabilityWindow) if err != nil { log.WithError(err).Error("poll db failed") } @@ -291,6 +325,6 @@ func (cb *CircuitBreakerManager) Run(ctx context.Context) { if err != nil { log.WithError(err).Error("Failed to sample events and update state") } - time.Sleep(30 * time.Second) + time.Sleep(time.Duration(cb.config.SampleTime) * time.Second) } } diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index b2c2a7f12c..7c0c32ef3a 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -39,7 +39,17 @@ func TestNewCircuitBreaker(t *testing.T) { testClock := clock.NewSimulatedClock(time.Now()) - b := NewCircuitBreakerManager(re.Client(), db.GetDB(), testClock) + c := CircuitBreakerConfig{ + SampleTime: 2, + ErrorTimeout: 30, + FailureThreshold: 10, + FailureCount: 1, + SuccessThreshold: 1, + ObservabilityWindow: 5, + NotificationThresholds: []int{10}, + ConsecutiveFailureThreshold: 10, + } + b := NewCircuitBreakerManager(re.Client(), db.GetDB(), testClock, c) endpointId := "endpoint-1" pollResults := [][]DBPollResult{ @@ -61,7 +71,7 @@ func TestNewCircuitBreaker(t *testing.T) { DBPollResult{ EndpointID: endpointId, Failures: 0, - Successes: 0, + Successes: 1, }, }, { @@ -77,12 +87,13 @@ func TestNewCircuitBreaker(t *testing.T) { innerErr := b.sampleEventsAndUpdateState(ctx, pollResults[i]) require.NoError(t, innerErr) - breakers, innerErr := b.loadCircuitBreakerStateFromRedis(ctx) - require.NoError(t, innerErr) + testClock.AdvanceTime(time.Minute) + } - require.Equal(t, len(breakers), 1) - t.Log(breakers) + breakers, innerErr := b.loadCircuitBreakerStateFromRedis(ctx) + require.NoError(t, innerErr) - testClock.AdvanceTime(time.Minute) + for i := 0; i < len(breakers); i++ { + require.Equal(t, breakers[i].State, StateClosed) } } From a4fc6645e3a646cf07f6a081c3dadb2e9eafd68e Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 19 Aug 2024 21:30:08 +0200 Subject: [PATCH 04/48] feat: add circuit breaker configuration --- cmd/worker/worker.go | 15 ++++- database/postgres/delivery_attempts.go | 28 +++++++++ datastore/repository.go | 2 + pkg/circuit_breaker/circuit_breaker.go | 70 +++++---------------- pkg/circuit_breaker/circuit_breaker_test.go | 10 +-- 5 files changed, 65 insertions(+), 60 deletions(-) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 098bf081e6..00111e7e7e 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -244,8 +244,19 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - breaker := circuit_breaker.NewCircuitBreakerManager(rd.Client(), a.DB.GetDB(), clock.NewRealClock()) - go breaker.Run(ctx) + // todo(raymond): fetch this config from the instance config + circuitBreakerConfig := circuit_breaker.CircuitBreakerConfig{ + SampleTime: 30, + ErrorTimeout: 30, + FailureThreshold: 10, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: []int{5, 10}, + ConsecutiveFailureThreshold: 10, + } + breaker := circuit_breaker.NewCircuitBreakerManager(rd.Client(), a.DB.GetDB(), clock.NewRealClock(), circuitBreakerConfig) + go breaker.Run(ctx, attemptRepo.GetFailureAndSuccessCounts) consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index 41d39d5723..bfbaeaed20 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -6,6 +6,7 @@ import ( "errors" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/pkg/circuit_breaker" "github.com/jmoiron/sqlx" "io" "time" @@ -130,6 +131,33 @@ func (d *deliveryAttemptRepo) DeleteProjectDeliveriesAttempts(ctx context.Contex return nil } +func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []circuit_breaker.PollResult, err error) { + query := ` + SELECT + endpoint_id, + COUNT(CASE WHEN status = false THEN 1 END) AS failures, + COUNT(CASE WHEN status = true THEN 1 END) AS successes + FROM convoy.delivery_attempts + WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) group by endpoint_id; + ` + + rows, err := d.db.QueryxContext(ctx, query, lookBackDuration) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var rowValue circuit_breaker.PollResult + if rowScanErr := rows.StructScan(&rowValue); rowScanErr != nil { + return nil, rowScanErr + } + results = append(results, rowValue) + } + + return results, nil +} + func (d *deliveryAttemptRepo) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { return exportRecords(ctx, d.db, "convoy.delivery_attempts", projectID, createdAt, w) } diff --git a/datastore/repository.go b/datastore/repository.go index cad2329a89..5f6bfd47ff 100644 --- a/datastore/repository.go +++ b/datastore/repository.go @@ -2,6 +2,7 @@ package datastore import ( "context" + "github.com/frain-dev/convoy/pkg/circuit_breaker" "io" "time" @@ -203,4 +204,5 @@ type DeliveryAttemptsRepository interface { FindDeliveryAttemptById(context.Context, string, string) (*DeliveryAttempt, error) FindDeliveryAttempts(context.Context, string) ([]DeliveryAttempt, error) DeleteProjectDeliveriesAttempts(ctx context.Context, projectID string, filter *DeliveryAttemptsFilter, hardDelete bool) error + GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []circuit_breaker.PollResult, err error) } diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 8882b41cfb..53c44b3ae3 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -21,18 +21,7 @@ var ( ErrOpenState = errors.New("circuit breaker is open") ) -// CircuitBreakerConfig is config which all the circuit breakers that manager manages will use -// -// { -// "sample_time": 5, -// "duration": 5, -// "error_timeout": 50, -// "error_threshold": 70, -// "failure_count": 10, -// "success_threshold": 10, -// "consecutive_failure_threshold": 10, -// "notification_thresholds": [30, 65] -// } +// CircuitBreakerConfig is the configuration that all the circuit breakers will use type CircuitBreakerConfig struct { // SampleTime is the time interval (in seconds) at which the data source // is polled to determine the number successful and failed requests @@ -53,7 +42,7 @@ type CircuitBreakerConfig struct { // after which a circuit breaker in the half-open state will go into the closed state SuccessThreshold int `json:"success_threshold"` - // ObservabilityWindow is how far back in time (in seconds) the data source is + // ObservabilityWindow is how far back in time (in minutes) the data source is // polled when determining the number successful and failed requests ObservabilityWindow int `json:"observability_window"` @@ -139,7 +128,7 @@ func (b *CircuitBreaker) resetCircuitBreaker() { b.ConsecutiveSuccesses++ } -type DBPollResult struct { +type PollResult struct { EndpointID string `json:"endpoint_id" db:"endpoint_id"` Failures int `json:"failures" db:"failures"` Successes int `json:"successes" db:"successes"` @@ -164,12 +153,12 @@ func NewCircuitBreakerManager(client redis.UniversalClient, db *sqlx.DB, clock c return r } -func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, dbPollResults []DBPollResult) error { +func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, pollResults []PollResult) error { var keys []string - for i := range dbPollResults { - key := fmt.Sprintf("%s:%s", prefix, dbPollResults[i].EndpointID) + for i := range pollResults { + key := fmt.Sprintf("%s:%s", prefix, pollResults[i].EndpointID) keys = append(keys, key) - dbPollResults[i].EndpointID = key + pollResults[i].EndpointID = key } res, err := cb.redis.MGet(context.Background(), keys...).Result() @@ -182,7 +171,7 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, if res[i] == nil { c := CircuitBreaker{ State: StateClosed, - EndpointID: dbPollResults[i].EndpointID, + EndpointID: pollResults[i].EndpointID, } circuitBreakers = append(circuitBreakers, c) continue @@ -198,8 +187,8 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, circuitBreakers = append(circuitBreakers, c) } - resultsMap := make(map[string]DBPollResult) - for _, result := range dbPollResults { + resultsMap := make(map[string]PollResult) + for _, result := range pollResults { resultsMap[result.EndpointID] = result } @@ -235,34 +224,6 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, return nil } -// todo(raymond): move this to the delivery attempts repo -func (cb *CircuitBreakerManager) getFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []DBPollResult, err error) { - query := ` - SELECT - endpoint_id, - COUNT(CASE WHEN status = false THEN 1 END) AS failures, - COUNT(CASE WHEN status = true THEN 1 END) AS successes - FROM convoy.delivery_attempts - WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) group by endpoint_id; - ` - - rows, err := cb.db.QueryxContext(ctx, query, lookBackDuration) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var rowValue DBPollResult - if rowScanErr := rows.StructScan(&rowValue); rowScanErr != nil { - return nil, rowScanErr - } - results = append(results, rowValue) - } - - return results, nil -} - func (cb *CircuitBreakerManager) updateCircuitBreakersInRedis(ctx context.Context, breakers map[string]CircuitBreaker) error { breakerStringsMap := make(map[string]string, len(breakers)) for key, breaker := range breakers { @@ -308,20 +269,23 @@ func (cb *CircuitBreakerManager) loadCircuitBreakerStateFromRedis(ctx context.Co return circuitBreakers, nil } -func (cb *CircuitBreakerManager) Run(ctx context.Context) { +func (cb *CircuitBreakerManager) Run(ctx context.Context, poolFunc func(ctx context.Context, lookBackDuration int) (results []PollResult, err error)) { for { // Get the failure and success counts from the last X minutes - dbPollResults, err := cb.getFailureAndSuccessCounts(ctx, cb.config.ObservabilityWindow) + pollResults, err := poolFunc(ctx, cb.config.ObservabilityWindow) if err != nil { log.WithError(err).Error("poll db failed") + time.Sleep(time.Duration(cb.config.SampleTime) * time.Second) + continue } - if len(dbPollResults) == 0 { + if len(pollResults) == 0 { // there's nothing to update + time.Sleep(time.Duration(cb.config.SampleTime) * time.Second) continue } - err = cb.sampleEventsAndUpdateState(ctx, dbPollResults) + err = cb.sampleEventsAndUpdateState(ctx, pollResults) if err != nil { log.WithError(err).Error("Failed to sample events and update state") } diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index 7c0c32ef3a..010eba01fa 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -52,30 +52,30 @@ func TestNewCircuitBreaker(t *testing.T) { b := NewCircuitBreakerManager(re.Client(), db.GetDB(), testClock, c) endpointId := "endpoint-1" - pollResults := [][]DBPollResult{ + pollResults := [][]PollResult{ { - DBPollResult{ + PollResult{ EndpointID: endpointId, Failures: 1, Successes: 0, }, }, { - DBPollResult{ + PollResult{ EndpointID: endpointId, Failures: 1, Successes: 0, }, }, { - DBPollResult{ + PollResult{ EndpointID: endpointId, Failures: 0, Successes: 1, }, }, { - DBPollResult{ + PollResult{ EndpointID: endpointId, Failures: 0, Successes: 1, From 528ef309d2663a06259a7adcee247dd7f5eca397 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 21 Aug 2024 12:10:59 +0200 Subject: [PATCH 05/48] feat: remove db depndency and add function to fetch breaker state --- cmd/worker/worker.go | 4 +- database/postgres/delivery_attempts.go | 2 +- mocks/repository.go | 16 +++++ pkg/circuit_breaker/circuit_breaker.go | 68 +++++++++++++------- pkg/circuit_breaker/circuit_breaker_test.go | 69 +++++++++------------ 5 files changed, 95 insertions(+), 64 deletions(-) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 00111e7e7e..5568a8fdd1 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -255,8 +255,8 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte NotificationThresholds: []int{5, 10}, ConsecutiveFailureThreshold: 10, } - breaker := circuit_breaker.NewCircuitBreakerManager(rd.Client(), a.DB.GetDB(), clock.NewRealClock(), circuitBreakerConfig) - go breaker.Run(ctx, attemptRepo.GetFailureAndSuccessCounts) + breaker := circuit_breaker.NewCircuitBreakerManager(rd.Client(), clock.NewRealClock(), circuitBreakerConfig) + go breaker.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index bfbaeaed20..cd39a7f03d 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -134,7 +134,7 @@ func (d *deliveryAttemptRepo) DeleteProjectDeliveriesAttempts(ctx context.Contex func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []circuit_breaker.PollResult, err error) { query := ` SELECT - endpoint_id, + endpoint_id AS key, COUNT(CASE WHEN status = false THEN 1 END) AS failures, COUNT(CASE WHEN status = true THEN 1 END) AS successes FROM convoy.delivery_attempts diff --git a/mocks/repository.go b/mocks/repository.go index 807456ccdb..86436148e2 100644 --- a/mocks/repository.go +++ b/mocks/repository.go @@ -16,6 +16,7 @@ import ( time "time" datastore "github.com/frain-dev/convoy/datastore" + circuit_breaker "github.com/frain-dev/convoy/pkg/circuit_breaker" flatten "github.com/frain-dev/convoy/pkg/flatten" gomock "go.uber.org/mock/gomock" ) @@ -2551,3 +2552,18 @@ func (mr *MockDeliveryAttemptsRepositoryMockRecorder) FindDeliveryAttempts(arg0, mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindDeliveryAttempts", reflect.TypeOf((*MockDeliveryAttemptsRepository)(nil).FindDeliveryAttempts), arg0, arg1) } + +// GetFailureAndSuccessCounts mocks base method. +func (m *MockDeliveryAttemptsRepository) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) ([]circuit_breaker.PollResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFailureAndSuccessCounts", ctx, lookBackDuration) + ret0, _ := ret[0].([]circuit_breaker.PollResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFailureAndSuccessCounts indicates an expected call of GetFailureAndSuccessCounts. +func (mr *MockDeliveryAttemptsRepositoryMockRecorder) GetFailureAndSuccessCounts(ctx, lookBackDuration any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFailureAndSuccessCounts", reflect.TypeOf((*MockDeliveryAttemptsRepository)(nil).GetFailureAndSuccessCounts), ctx, lookBackDuration) +} diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 53c44b3ae3..6c0d363122 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -7,7 +7,6 @@ import ( "github.com/frain-dev/convoy/pkg/clock" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/pkg/msgpack" - "github.com/jmoiron/sqlx" "github.com/redis/go-redis/v9" "time" ) @@ -17,8 +16,12 @@ const prefix = "breaker" var ( // ErrTooManyRequests is returned when the CB state is half open and the requests count is over the cb maxRequests ErrTooManyRequests = errors.New("too many requests") + // ErrOpenState is returned when the CB state is open ErrOpenState = errors.New("circuit breaker is open") + + // ErrCircuitBreakerNotFound is returned when the circuit breaker is not found + ErrCircuitBreakerNotFound = errors.New("circuit breaker not found") ) // CircuitBreakerConfig is the configuration that all the circuit breakers will use @@ -92,9 +95,9 @@ func (s State) String() string { // CircuitBreaker represents a circuit breaker // todo(raymond): implement methods to check the state and find out if an action can be performed. type CircuitBreaker struct { + Key string `json:"key"` State State `json:"state"` Requests int `json:"requests"` - EndpointID string `json:"endpoint_id"` WillResetAt time.Time `json:"will_reset_at"` TotalFailures int `json:"total_failures"` TotalSuccesses int `json:"total_successes"` @@ -129,9 +132,9 @@ func (b *CircuitBreaker) resetCircuitBreaker() { } type PollResult struct { - EndpointID string `json:"endpoint_id" db:"endpoint_id"` - Failures int `json:"failures" db:"failures"` - Successes int `json:"successes" db:"successes"` + Key string `json:"key" db:"key"` + Failures int `json:"failures" db:"failures"` + Successes int `json:"successes" db:"successes"` } type CircuitBreakerManager struct { @@ -139,12 +142,10 @@ type CircuitBreakerManager struct { config CircuitBreakerConfig clock clock.Clock redis *redis.Client - db *sqlx.DB } -func NewCircuitBreakerManager(client redis.UniversalClient, db *sqlx.DB, clock clock.Clock, config CircuitBreakerConfig) *CircuitBreakerManager { +func NewCircuitBreakerManager(client redis.UniversalClient, clock clock.Clock, config CircuitBreakerConfig) *CircuitBreakerManager { r := &CircuitBreakerManager{ - db: db, clock: clock, config: config, redis: client.(*redis.Client), @@ -153,12 +154,12 @@ func NewCircuitBreakerManager(client redis.UniversalClient, db *sqlx.DB, clock c return r } -func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, pollResults []PollResult) error { +func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { var keys []string for i := range pollResults { - key := fmt.Sprintf("%s:%s", prefix, pollResults[i].EndpointID) + key := fmt.Sprintf("%s:%s", prefix, pollResults[i].Key) keys = append(keys, key) - pollResults[i].EndpointID = key + pollResults[i].Key = key } res, err := cb.redis.MGet(context.Background(), keys...).Result() @@ -170,8 +171,8 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, for i := range res { if res[i] == nil { c := CircuitBreaker{ - State: StateClosed, - EndpointID: pollResults[i].EndpointID, + State: StateClosed, + Key: pollResults[i].Key, } circuitBreakers = append(circuitBreakers, c) continue @@ -189,13 +190,13 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, resultsMap := make(map[string]PollResult) for _, result := range pollResults { - resultsMap[result.EndpointID] = result + resultsMap[result.Key] = result } circuitBreakerMap := make(map[string]CircuitBreaker, len(resultsMap)) for _, breaker := range circuitBreakers { - result := resultsMap[breaker.EndpointID] + result := resultsMap[breaker.Key] breaker.Requests = result.Successes + result.Failures breaker.TotalFailures = result.Failures @@ -213,18 +214,18 @@ func (cb *CircuitBreakerManager) sampleEventsAndUpdateState(ctx context.Context, breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.ErrorTimeout) * time.Second)) } - circuitBreakerMap[breaker.EndpointID] = breaker + circuitBreakerMap[breaker.Key] = breaker } - // Update the circuit breaker state in Redis - if err = cb.updateCircuitBreakersInRedis(ctx, circuitBreakerMap); err != nil { - log.WithError(err).Error("Failed to update state in Redis") + if err = cb.updateCircuitBreakers(ctx, circuitBreakerMap); err != nil { + log.WithError(err).Error("failed to update state") + return err } return nil } -func (cb *CircuitBreakerManager) updateCircuitBreakersInRedis(ctx context.Context, breakers map[string]CircuitBreaker) error { +func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) error { breakerStringsMap := make(map[string]string, len(breakers)) for key, breaker := range breakers { val, err := breaker.String() @@ -243,7 +244,7 @@ func (cb *CircuitBreakerManager) updateCircuitBreakersInRedis(ctx context.Contex return nil } -func (cb *CircuitBreakerManager) loadCircuitBreakerStateFromRedis(ctx context.Context) ([]CircuitBreaker, error) { +func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { keys, err := cb.redis.Keys(ctx, "breaker*").Result() if err != nil { return nil, err @@ -269,7 +270,28 @@ func (cb *CircuitBreakerManager) loadCircuitBreakerStateFromRedis(ctx context.Co return circuitBreakers, nil } -func (cb *CircuitBreakerManager) Run(ctx context.Context, poolFunc func(ctx context.Context, lookBackDuration int) (results []PollResult, err error)) { +// GetCircuitBreaker is used to get fetch the circuit breaker state before executing a function +func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { + breakerKey := fmt.Sprintf("%s:%s", prefix, key) + + res, err := cb.redis.Get(ctx, breakerKey).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return nil, err + } + + if err != nil && errors.Is(err, redis.Nil) { + return nil, ErrCircuitBreakerNotFound + } + + err = msgpack.DecodeMsgPack([]byte(res), &c) + if err != nil { + return nil, err + } + + return c, nil +} + +func (cb *CircuitBreakerManager) Start(ctx context.Context, poolFunc func(ctx context.Context, lookBackDuration int) (results []PollResult, err error)) { for { // Get the failure and success counts from the last X minutes pollResults, err := poolFunc(ctx, cb.config.ObservabilityWindow) @@ -285,7 +307,7 @@ func (cb *CircuitBreakerManager) Run(ctx context.Context, poolFunc func(ctx cont continue } - err = cb.sampleEventsAndUpdateState(ctx, pollResults) + err = cb.sampleStore(ctx, pollResults) if err != nil { log.WithError(err).Error("Failed to sample events and update state") } diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index 010eba01fa..7b5ead503d 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -2,39 +2,34 @@ package circuit_breaker import ( "context" - "github.com/frain-dev/convoy/config" - "github.com/frain-dev/convoy/database/postgres" - "github.com/frain-dev/convoy/internal/pkg/rdb" "github.com/frain-dev/convoy/pkg/clock" + "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "testing" "time" ) +func getRedis(t *testing.T) (client redis.UniversalClient, err error) { + t.Helper() + + opts, err := redis.ParseURL("redis://localhost:6379") + if err != nil { + return nil, err + } + + return redis.NewClient(opts), nil +} + func TestNewCircuitBreaker(t *testing.T) { ctx := context.Background() - re, err := rdb.NewClient([]string{"redis://localhost:6379"}) + re, err := getRedis(t) require.NoError(t, err) - keys, err := re.Client().Keys(ctx, "breaker*").Result() + keys, err := re.Keys(ctx, "breaker*").Result() require.NoError(t, err) - err = re.Client().Del(ctx, keys...).Err() - require.NoError(t, err) - - db, err := postgres.NewDB(config.Configuration{ - Database: config.DatabaseConfiguration{ - Type: config.PostgresDatabaseProvider, - Scheme: "postgres", - Host: "localhost", - Username: "postgres", - Password: "postgres", - Database: "endpoint_fix", - Options: "sslmode=disable&connect_timeout=30", - Port: 5432, - }, - }) + err = re.Del(ctx, keys...).Err() require.NoError(t, err) testClock := clock.NewSimulatedClock(time.Now()) @@ -49,51 +44,49 @@ func TestNewCircuitBreaker(t *testing.T) { NotificationThresholds: []int{10}, ConsecutiveFailureThreshold: 10, } - b := NewCircuitBreakerManager(re.Client(), db.GetDB(), testClock, c) + b := NewCircuitBreakerManager(re, testClock, c) endpointId := "endpoint-1" pollResults := [][]PollResult{ { PollResult{ - EndpointID: endpointId, - Failures: 1, - Successes: 0, + Key: endpointId, + Failures: 1, + Successes: 0, }, }, { PollResult{ - EndpointID: endpointId, - Failures: 1, - Successes: 0, + Key: endpointId, + Failures: 1, + Successes: 0, }, }, { PollResult{ - EndpointID: endpointId, - Failures: 0, - Successes: 1, + Key: endpointId, + Failures: 0, + Successes: 1, }, }, { PollResult{ - EndpointID: endpointId, - Failures: 0, - Successes: 1, + Key: endpointId, + Failures: 0, + Successes: 1, }, }, } for i := 0; i < len(pollResults); i++ { - innerErr := b.sampleEventsAndUpdateState(ctx, pollResults[i]) + innerErr := b.sampleStore(ctx, pollResults[i]) require.NoError(t, innerErr) testClock.AdvanceTime(time.Minute) } - breakers, innerErr := b.loadCircuitBreakerStateFromRedis(ctx) + breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) require.NoError(t, innerErr) - for i := 0; i < len(breakers); i++ { - require.Equal(t, breakers[i].State, StateClosed) - } + require.Equal(t, breaker.State, StateClosed) } From feb2b229022cc5282171ec6266223127bf6c262f Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 21 Aug 2024 17:28:35 +0200 Subject: [PATCH 06/48] feat: rename sample_time to sample rate; set reasonable defaults when creating the circuit breaker manager instance --- pkg/circuit_breaker/circuit_breaker.go | 40 +++++++++++---- pkg/circuit_breaker/circuit_breaker_test.go | 57 ++++++++++++--------- pkg/clock/clock.go | 4 +- 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 6c0d363122..d5fb632df5 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -26,9 +26,9 @@ var ( // CircuitBreakerConfig is the configuration that all the circuit breakers will use type CircuitBreakerConfig struct { - // SampleTime is the time interval (in seconds) at which the data source + // SampleRate is the time interval (in seconds) at which the data source // is polled to determine the number successful and failed requests - SampleTime int `json:"sample_time"` + SampleRate int `json:"sample_rate"` // ErrorTimeout is the time (in seconds) after which a circuit breaker goes // into the half-open state from the open state @@ -93,7 +93,6 @@ func (s State) String() string { } // CircuitBreaker represents a circuit breaker -// todo(raymond): implement methods to check the state and find out if an action can be performed. type CircuitBreaker struct { Key string `json:"key"` State State `json:"state"` @@ -138,22 +137,43 @@ type PollResult struct { } type CircuitBreakerManager struct { + config *CircuitBreakerConfig breakers []CircuitBreaker - config CircuitBreakerConfig clock clock.Clock redis *redis.Client } -func NewCircuitBreakerManager(client redis.UniversalClient, clock clock.Clock, config CircuitBreakerConfig) *CircuitBreakerManager { +func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManager { + defaultConfig := &CircuitBreakerConfig{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: []int{5, 10}, + ConsecutiveFailureThreshold: 10, + } + r := &CircuitBreakerManager{ - clock: clock, - config: config, + config: defaultConfig, + clock: clock.NewRealClock(), redis: client.(*redis.Client), } return r } +func (cb *CircuitBreakerManager) WithClock(c clock.Clock) *CircuitBreakerManager { + cb.clock = c + return cb +} + +func (cb *CircuitBreakerManager) WithConfig(config *CircuitBreakerConfig) *CircuitBreakerManager { + cb.config = config + return cb +} + func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { var keys []string for i := range pollResults { @@ -297,13 +317,13 @@ func (cb *CircuitBreakerManager) Start(ctx context.Context, poolFunc func(ctx co pollResults, err := poolFunc(ctx, cb.config.ObservabilityWindow) if err != nil { log.WithError(err).Error("poll db failed") - time.Sleep(time.Duration(cb.config.SampleTime) * time.Second) + time.Sleep(time.Duration(cb.config.SampleRate) * time.Second) continue } if len(pollResults) == 0 { // there's nothing to update - time.Sleep(time.Duration(cb.config.SampleTime) * time.Second) + time.Sleep(time.Duration(cb.config.SampleRate) * time.Second) continue } @@ -311,6 +331,6 @@ func (cb *CircuitBreakerManager) Start(ctx context.Context, poolFunc func(ctx co if err != nil { log.WithError(err).Error("Failed to sample events and update state") } - time.Sleep(time.Duration(cb.config.SampleTime) * time.Second) + time.Sleep(time.Duration(cb.config.SampleRate) * time.Second) } } diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index 7b5ead503d..d583bf3db5 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -20,6 +20,16 @@ func getRedis(t *testing.T) (client redis.UniversalClient, err error) { return redis.NewClient(opts), nil } +func pollResult(t *testing.T, key string, failureCount, successCount int) PollResult { + t.Helper() + + return PollResult{ + Key: key, + Failures: failureCount, + Successes: successCount, + } +} + func TestNewCircuitBreaker(t *testing.T) { ctx := context.Background() @@ -34,47 +44,40 @@ func TestNewCircuitBreaker(t *testing.T) { testClock := clock.NewSimulatedClock(time.Now()) - c := CircuitBreakerConfig{ - SampleTime: 2, + c := &CircuitBreakerConfig{ + SampleRate: 2, ErrorTimeout: 30, - FailureThreshold: 10, - FailureCount: 1, + FailureThreshold: 0.1, + FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, NotificationThresholds: []int{10}, ConsecutiveFailureThreshold: 10, } - b := NewCircuitBreakerManager(re, testClock, c) + b := NewCircuitBreakerManager(re).WithClock(testClock).WithConfig(c) endpointId := "endpoint-1" pollResults := [][]PollResult{ { - PollResult{ - Key: endpointId, - Failures: 1, - Successes: 0, - }, + pollResult(t, endpointId, 1, 0), + }, + { + pollResult(t, endpointId, 2, 0), }, { - PollResult{ - Key: endpointId, - Failures: 1, - Successes: 0, - }, + pollResult(t, endpointId, 2, 1), }, { - PollResult{ - Key: endpointId, - Failures: 0, - Successes: 1, - }, + pollResult(t, endpointId, 2, 2), }, { - PollResult{ - Key: endpointId, - Failures: 0, - Successes: 1, - }, + pollResult(t, endpointId, 2, 3), + }, + { + pollResult(t, endpointId, 1, 4), + }, + { + pollResult(t, endpointId, 0, 5), }, } @@ -82,6 +85,10 @@ func TestNewCircuitBreaker(t *testing.T) { innerErr := b.sampleStore(ctx, pollResults[i]) require.NoError(t, innerErr) + breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) + require.NoError(t, innerErr) + t.Logf("%+v\n", breaker) + testClock.AdvanceTime(time.Minute) } diff --git a/pkg/clock/clock.go b/pkg/clock/clock.go index c2a79dd97f..26884a1485 100644 --- a/pkg/clock/clock.go +++ b/pkg/clock/clock.go @@ -27,12 +27,12 @@ func (_ *realTimeClock) Now() time.Time { return time.Now() } // Time is advanced by explicit call to the AdvanceTime() or SetTime() functions. // This object is concurrency safe. type SimulatedClock struct { - mu sync.Mutex + mu *sync.Mutex t time.Time // guarded by mu } func NewSimulatedClock(t time.Time) *SimulatedClock { - return &SimulatedClock{t: t} + return &SimulatedClock{mu: &sync.Mutex{}, t: t} } func (c *SimulatedClock) Now() time.Time { From e96034eddbc24c42e9f52dbb68b7c16e2f2226d1 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 21 Aug 2024 17:30:27 +0200 Subject: [PATCH 07/48] feat: add circuit breaker to instance config --- api/handlers/configuration.go | 26 ++++++------- api/testdb/seed.go | 16 ++++---- cmd/hooks/hooks.go | 36 +++++++++++++----- cmd/main.go | 4 +- cmd/worker/worker.go | 16 +------- config/config.go | 22 +++++++++++ database/postgres/configuration.go | 42 ++++++++++++++++++++- datastore/models.go | 60 +++++++++++++++++++++++++++--- sql/1724236900.sql | 19 ++++++++++ 9 files changed, 189 insertions(+), 52 deletions(-) create mode 100644 sql/1724236900.sql diff --git a/api/handlers/configuration.go b/api/handlers/configuration.go index ad0d51ee25..c7e6dddcb8 100644 --- a/api/handlers/configuration.go +++ b/api/handlers/configuration.go @@ -17,24 +17,24 @@ import ( ) func (h *Handler) GetConfiguration(w http.ResponseWriter, r *http.Request) { - config, err := postgres.NewConfigRepo(h.A.DB).LoadConfiguration(r.Context()) + configuration, err := postgres.NewConfigRepo(h.A.DB).LoadConfiguration(r.Context()) if err != nil && !errors.Is(err, datastore.ErrConfigNotFound) { _ = render.Render(w, r, util.NewServiceErrResponse(err)) return } - configResponse := []*models.ConfigurationResponse{} - if config != nil { - if config.StoragePolicy.Type == datastore.S3 { + var configResponse []*models.ConfigurationResponse + if configuration != nil { + if configuration.StoragePolicy.Type == datastore.S3 { policy := &datastore.S3Storage{} - policy.Bucket = config.StoragePolicy.S3.Bucket - policy.Endpoint = config.StoragePolicy.S3.Endpoint - policy.Region = config.StoragePolicy.S3.Region - config.StoragePolicy.S3 = policy + policy.Bucket = configuration.StoragePolicy.S3.Bucket + policy.Endpoint = configuration.StoragePolicy.S3.Endpoint + policy.Region = configuration.StoragePolicy.S3.Region + configuration.StoragePolicy.S3 = policy } c := &models.ConfigurationResponse{ - Configuration: config, + Configuration: configuration, ApiVersion: convoy.GetVersion(), } @@ -61,14 +61,14 @@ func (h *Handler) CreateConfiguration(w http.ResponseWriter, r *http.Request) { NewConfig: &newConfig, } - config, err := cc.Run(r.Context()) + configuration, err := cc.Run(r.Context()) if err != nil { _ = render.Render(w, r, util.NewServiceErrResponse(err)) return } c := &models.ConfigurationResponse{ - Configuration: config, + Configuration: configuration, ApiVersion: convoy.GetVersion(), } @@ -92,14 +92,14 @@ func (h *Handler) UpdateConfiguration(w http.ResponseWriter, r *http.Request) { Config: &newConfig, } - config, err := uc.Run(r.Context()) + configuration, err := uc.Run(r.Context()) if err != nil { _ = render.Render(w, r, util.NewServiceErrResponse(err)) return } c := &models.ConfigurationResponse{ - Configuration: config, + Configuration: configuration, ApiVersion: convoy.GetVersion(), } diff --git a/api/testdb/seed.go b/api/testdb/seed.go index 728744f6b3..4a7d25d4cd 100644 --- a/api/testdb/seed.go +++ b/api/testdb/seed.go @@ -557,21 +557,23 @@ func SeedUser(db database.Database, email, password string) (*datastore.User, er } func SeedConfiguration(db database.Database) (*datastore.Configuration, error) { - config := &datastore.Configuration{ - UID: ulid.Make().String(), - IsAnalyticsEnabled: true, - IsSignupEnabled: true, - StoragePolicy: &datastore.DefaultStoragePolicy, + c := &datastore.Configuration{ + UID: ulid.Make().String(), + IsAnalyticsEnabled: true, + IsSignupEnabled: true, + StoragePolicy: &datastore.DefaultStoragePolicy, + RetentionPolicy: &datastore.DefaultRetentionPolicy, + CircuitBreakerConfig: &datastore.DefaultCircuitBreakerConfiguration, } // Seed Data configRepo := postgres.NewConfigRepo(db) - err := configRepo.CreateConfiguration(context.TODO(), config) + err := configRepo.CreateConfiguration(context.TODO(), c) if err != nil { return nil, err } - return config, nil + return c, nil } func SeedDevice(db database.Database, g *datastore.Project, endpointID string) error { diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index 5dfbba6d58..a370d57265 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/lib/pq" "io" "os" "time" @@ -283,21 +284,37 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat IsRetentionPolicyEnabled: cfg.RetentionPolicy.IsRetentionPolicyEnabled, } + notificationThresholds := pq.Int64Array{} + for i := range cfg.CircuitBreaker.NotificationThresholds { + notificationThresholds = append(notificationThresholds, int64(cfg.CircuitBreaker.NotificationThresholds[i])) + } + circuitBreakerConfig := &datastore.CircuitBreakerConfig{ + SampleRate: cfg.CircuitBreaker.SampleRate, + ErrorTimeout: cfg.CircuitBreaker.ErrorTimeout, + FailureCount: cfg.CircuitBreaker.FailureCount, + FailureThreshold: cfg.CircuitBreaker.FailureThreshold, + SuccessThreshold: cfg.CircuitBreaker.SuccessThreshold, + ObservabilityWindow: cfg.CircuitBreaker.ObservabilityWindow, + NotificationThresholds: notificationThresholds, + ConsecutiveFailureThreshold: cfg.CircuitBreaker.ConsecutiveFailureThreshold, + } + configuration, err := configRepo.LoadConfiguration(ctx) if err != nil { if errors.Is(err, datastore.ErrConfigNotFound) { a.Logger.Info("Creating Instance Config") - cfg := &datastore.Configuration{ - UID: ulid.Make().String(), - StoragePolicy: storagePolicy, - IsAnalyticsEnabled: cfg.Analytics.IsEnabled, - IsSignupEnabled: cfg.Auth.IsSignupEnabled, - RetentionPolicy: retentionPolicy, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + c := &datastore.Configuration{ + UID: ulid.Make().String(), + StoragePolicy: storagePolicy, + IsAnalyticsEnabled: cfg.Analytics.IsEnabled, + IsSignupEnabled: cfg.Auth.IsSignupEnabled, + RetentionPolicy: retentionPolicy, + CircuitBreakerConfig: circuitBreakerConfig, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), } - return cfg, configRepo.CreateConfiguration(ctx, cfg) + return c, configRepo.CreateConfiguration(ctx, c) } return configuration, err @@ -306,6 +323,7 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat configuration.StoragePolicy = storagePolicy configuration.IsSignupEnabled = cfg.Auth.IsSignupEnabled configuration.IsAnalyticsEnabled = cfg.Analytics.IsEnabled + configuration.CircuitBreakerConfig = circuitBreakerConfig configuration.RetentionPolicy = retentionPolicy configuration.UpdatedAt = time.Now() diff --git a/cmd/main.go b/cmd/main.go index c10ac21ae3..17f1e9f361 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -110,8 +110,8 @@ func main() { c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('experimental' feature flag level required") c.Flags().Uint64Var(&prometheusMetricsSampleTime, "metrics-prometheus-sample-time", 5, "Prometheus metrics sample time") - c.Flags().StringVar(&retentionPolicy, "retention-policy", "", "SMTP Port") - c.Flags().BoolVar(&retentionPolicyEnabled, "retention-policy-enabled", false, "SMTP Port") + c.Flags().StringVar(&retentionPolicy, "retention-policy", "", "Retention Policy Duration") + c.Flags().BoolVar(&retentionPolicyEnabled, "retention-policy-enabled", false, "Retention Policy Enabled") c.Flags().Uint64Var(&maxRetrySeconds, "max-retry-seconds", 7200, "Max retry seconds exponential backoff") diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 5568a8fdd1..cbd01d56d2 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -17,7 +17,6 @@ import ( "github.com/frain-dev/convoy/internal/telemetry" "github.com/frain-dev/convoy/net" "github.com/frain-dev/convoy/pkg/circuit_breaker" - "github.com/frain-dev/convoy/pkg/clock" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/queue" redisQueue "github.com/frain-dev/convoy/queue/redis" @@ -244,19 +243,8 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - // todo(raymond): fetch this config from the instance config - circuitBreakerConfig := circuit_breaker.CircuitBreakerConfig{ - SampleTime: 30, - ErrorTimeout: 30, - FailureThreshold: 10, - FailureCount: 10, - SuccessThreshold: 5, - ObservabilityWindow: 5, - NotificationThresholds: []int{5, 10}, - ConsecutiveFailureThreshold: 10, - } - breaker := circuit_breaker.NewCircuitBreakerManager(rd.Client(), clock.NewRealClock(), circuitBreakerConfig) - go breaker.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) + circuitBreakerManager := circuit_breaker.NewCircuitBreakerManager(rd.Client()).WithConfig(configuration.ToCircuitBreakerConfig()) + go circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, diff --git a/config/config.go b/config/config.go index 4b023e3367..ceb32b5ffa 100644 --- a/config/config.go +++ b/config/config.go @@ -74,6 +74,16 @@ var DefaultConfiguration = Configuration{ Policy: "720h", IsRetentionPolicyEnabled: false, }, + CircuitBreaker: CircuitBreakerConfiguration{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: []int{5, 10}, + ConsecutiveFailureThreshold: 10, + }, Auth: AuthConfiguration{ IsSignupEnabled: true, Native: NativeRealmOptions{ @@ -261,6 +271,17 @@ type RetentionPolicyConfiguration struct { IsRetentionPolicyEnabled bool `json:"enabled" envconfig:"CONVOY_RETENTION_POLICY_ENABLED"` } +type CircuitBreakerConfiguration struct { + SampleRate int `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` + FailureCount int `json:"failure_count" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_COUNT"` + ErrorTimeout int `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` + FailureThreshold float64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` + SuccessThreshold int `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` + ObservabilityWindow int `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` + NotificationThresholds []int `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` + ConsecutiveFailureThreshold int `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` +} + type AnalyticsConfiguration struct { IsEnabled bool `json:"enabled" envconfig:"CONVOY_ANALYTICS_ENABLED"` } @@ -383,6 +404,7 @@ type Configuration struct { CustomDomainSuffix string `json:"custom_domain_suffix" envconfig:"CONVOY_CUSTOM_DOMAIN_SUFFIX"` FeatureFlag FlagLevel `json:"feature_flag" envconfig:"CONVOY_FEATURE_FLAG"` RetentionPolicy RetentionPolicyConfiguration `json:"retention_policy"` + CircuitBreaker CircuitBreakerConfiguration `json:"circuit_breaker"` Analytics AnalyticsConfiguration `json:"analytics"` StoragePolicy StoragePolicyConfiguration `json:"storage_policy"` ConsumerPoolSize int `json:"consumer_pool_size" envconfig:"CONVOY_CONSUMER_POOL_SIZE"` diff --git a/database/postgres/configuration.go b/database/postgres/configuration.go index 30fcbf0e01..8579c10d28 100644 --- a/database/postgres/configuration.go +++ b/database/postgres/configuration.go @@ -19,9 +19,13 @@ const ( storage_policy_type, on_prem_path, s3_prefix, s3_bucket, s3_access_key, s3_secret_key, s3_region, s3_session_token, s3_endpoint, - retention_policy_policy, retention_policy_enabled + retention_policy_policy, retention_policy_enabled, + cb_sample_rate,cb_error_timeout, + cb_failure_threshold,cb_failure_count, + cb_success_threshold,cb_observability_window, + cb_notification_thresholds,cb_consecutive_failure_threshold ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14); + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22); ` fetchConfiguration = ` @@ -40,6 +44,14 @@ const ( s3_session_token AS "storage_policy.s3.session_token", s3_endpoint AS "storage_policy.s3.endpoint", s3_prefix AS "storage_policy.s3.prefix", + cb_sample_rate AS "breaker_config.sample_rate", + cb_error_timeout AS "breaker_config.error_timeout", + cb_failure_threshold AS "breaker_config.failure_threshold", + cb_failure_count AS "breaker_config.failure_count", + cb_success_threshold AS "breaker_config.success_threshold", + cb_observability_window AS "breaker_config.observability_window", + cb_notification_thresholds::INTEGER[] AS "breaker_config.notification_thresholds", + cb_consecutive_failure_threshold AS "breaker_config.consecutive_failure_threshold", created_at, updated_at, deleted_at @@ -64,6 +76,14 @@ const ( s3_prefix = $12, retention_policy_policy = $13, retention_policy_enabled = $14, + cb_sample_rate = $15, + cb_error_timeout = $16, + cb_failure_threshold = $17, + cb_failure_count = $18, + cb_success_threshold = $19, + cb_observability_window = $20, + cb_notification_thresholds = $21, + cb_consecutive_failure_threshold = $22, updated_at = NOW() WHERE id = $1 AND deleted_at IS NULL; ` @@ -95,6 +115,7 @@ func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore. } rc := config.GetRetentionPolicyConfig() + cb := config.GetCircuitBreakerConfig() r, err := c.db.ExecContext(ctx, createConfiguration, config.UID, @@ -111,6 +132,14 @@ func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore. config.StoragePolicy.S3.Endpoint, rc.Policy, rc.IsRetentionPolicyEnabled, + cb.SampleRate, + cb.ErrorTimeout, + cb.FailureThreshold, + cb.FailureCount, + cb.SuccessThreshold, + cb.ObservabilityWindow, + cb.NotificationThresholds, + cb.ConsecutiveFailureThreshold, ) if err != nil { return err @@ -159,6 +188,7 @@ func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Con } rc := cfg.GetRetentionPolicyConfig() + cb := cfg.GetCircuitBreakerConfig() result, err := c.db.ExecContext(ctx, updateConfiguration, cfg.UID, @@ -175,6 +205,14 @@ func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Con cfg.StoragePolicy.S3.Prefix, rc.Policy, rc.IsRetentionPolicyEnabled, + cb.SampleRate, + cb.ErrorTimeout, + cb.FailureThreshold, + cb.FailureCount, + cb.SuccessThreshold, + cb.ObservabilityWindow, + cb.NotificationThresholds, + cb.ConsecutiveFailureThreshold, ) if err != nil { return err diff --git a/datastore/models.go b/datastore/models.go index c6a3d4a774..a3ea4378d5 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/frain-dev/convoy/pkg/circuit_breaker" "math" "net/http" "strings" @@ -317,6 +318,17 @@ var ( IsRetentionPolicyEnabled: false, Policy: "720h", } + + DefaultCircuitBreakerConfiguration = CircuitBreakerConfig{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: pq.Int64Array{5, 10}, + ConsecutiveFailureThreshold: 10, + } ) func GetDefaultSignatureConfig() *SignatureConfiguration { @@ -1312,17 +1324,44 @@ type Organisation struct { } type Configuration struct { - UID string `json:"uid" db:"id"` - IsAnalyticsEnabled bool `json:"is_analytics_enabled" db:"is_analytics_enabled"` - IsSignupEnabled bool `json:"is_signup_enabled" db:"is_signup_enabled"` - StoragePolicy *StoragePolicyConfiguration `json:"storage_policy" db:"storage_policy"` - RetentionPolicy *RetentionPolicyConfiguration `json:"retention_policy" db:"retention_policy"` + UID string `json:"uid" db:"id"` + IsAnalyticsEnabled bool `json:"is_analytics_enabled" db:"is_analytics_enabled"` + IsSignupEnabled bool `json:"is_signup_enabled" db:"is_signup_enabled"` + + StoragePolicy *StoragePolicyConfiguration `json:"storage_policy" db:"storage_policy"` + RetentionPolicy *RetentionPolicyConfiguration `json:"retention_policy" db:"retention_policy"` + CircuitBreakerConfig *CircuitBreakerConfig `json:"breaker_config" db:"breaker_config"` CreatedAt time.Time `json:"created_at,omitempty" db:"created_at,omitempty" swaggertype:"string"` UpdatedAt time.Time `json:"updated_at,omitempty" db:"updated_at,omitempty" swaggertype:"string"` DeletedAt null.Time `json:"deleted_at,omitempty" db:"deleted_at" swaggertype:"string"` } +func (c *Configuration) GetCircuitBreakerConfig() CircuitBreakerConfig { + if c.CircuitBreakerConfig != nil { + return *c.CircuitBreakerConfig + } + return CircuitBreakerConfig{} +} + +func (c *Configuration) ToCircuitBreakerConfig() *circuit_breaker.CircuitBreakerConfig { + notificationThresholds := make([]int, len(c.CircuitBreakerConfig.NotificationThresholds)) + for i := range c.CircuitBreakerConfig.NotificationThresholds { + notificationThresholds[i] = int(c.CircuitBreakerConfig.NotificationThresholds[i]) + } + + return &circuit_breaker.CircuitBreakerConfig{ + SampleRate: c.CircuitBreakerConfig.SampleRate, + ErrorTimeout: c.CircuitBreakerConfig.ErrorTimeout, + FailureThreshold: c.CircuitBreakerConfig.FailureThreshold, + FailureCount: c.CircuitBreakerConfig.FailureCount, + SuccessThreshold: c.CircuitBreakerConfig.SuccessThreshold, + ObservabilityWindow: c.CircuitBreakerConfig.ObservabilityWindow, + NotificationThresholds: notificationThresholds, + ConsecutiveFailureThreshold: c.CircuitBreakerConfig.ConsecutiveFailureThreshold, + } +} + func (c *Configuration) GetRetentionPolicyConfig() RetentionPolicyConfiguration { if c.RetentionPolicy != nil { return *c.RetentionPolicy @@ -1350,6 +1389,17 @@ type OnPremStorage struct { Path null.String `json:"path" db:"path"` } +type CircuitBreakerConfig struct { + SampleRate int `json:"sample_rate" db:"sample_rate"` + ErrorTimeout int `json:"error_timeout" db:"error_timeout"` + FailureThreshold float64 `json:"failure_threshold" db:"failure_threshold"` + FailureCount int `json:"failure_count" db:"failure_count"` + SuccessThreshold int `json:"success_threshold" db:"success_threshold"` + ObservabilityWindow int `json:"observability_window" db:"observability_window"` + NotificationThresholds pq.Int64Array `json:"notification_thresholds" db:"notification_thresholds"` + ConsecutiveFailureThreshold int `json:"consecutive_failure_threshold" db:"consecutive_failure_threshold"` +} + type OrganisationMember struct { UID string `json:"uid" db:"id"` OrganisationID string `json:"organisation_id" db:"organisation_id"` diff --git a/sql/1724236900.sql b/sql/1724236900.sql new file mode 100644 index 0000000000..6bd47e43e7 --- /dev/null +++ b/sql/1724236900.sql @@ -0,0 +1,19 @@ +-- +migrate Up +alter table convoy.configurations add column if not exists cb_sample_rate int not null default 30; +alter table convoy.configurations add column if not exists cb_error_timeout int not null default 30; +alter table convoy.configurations add column if not exists cb_failure_threshold float not null default 0.1; +alter table convoy.configurations add column if not exists cb_failure_count int not null default 1; +alter table convoy.configurations add column if not exists cb_success_threshold int not null default 5; +alter table convoy.configurations add column if not exists cb_observability_window int not null default 5; +alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[5, 10]; +alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 5; + +-- +migrate Down +alter table convoy.configurations drop column if exists cb_sample_rate; +alter table convoy.configurations drop column if exists cb_error_timeout; +alter table convoy.configurations drop column if exists cb_failure_threshold; +alter table convoy.configurations drop column if exists cb_failure_count; +alter table convoy.configurations drop column if exists cb_success_threshold; +alter table convoy.configurations drop column if exists cb_observability_window; +alter table convoy.configurations drop column if exists cb_notification_thresholds; +alter table convoy.configurations drop column if exists cb_consecutive_failure_threshold; From be3208e6079848cfda0589c69c55b842e00c3e19 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Thu, 22 Aug 2024 08:53:50 +0200 Subject: [PATCH 08/48] feat: move half-open decision after other checks have been made, so we don't immediately go from half-open to closed in one loop iteration --- pkg/circuit_breaker/circuit_breaker.go | 20 +++++++------------- pkg/circuit_breaker/circuit_breaker_test.go | 5 +---- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index d5fb632df5..75b76dcb50 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -223,17 +223,16 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] breaker.TotalSuccesses = result.Successes breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.TotalSuccesses+breaker.TotalFailures) - // todo(raymond): move this to a different place that runs in a goroutine - if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { - breaker.toHalfOpen() - } - if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { breaker.resetCircuitBreaker() } else if breaker.State == StateClosed && (breaker.FailureRate >= cb.config.FailureThreshold || breaker.TotalFailures >= cb.config.FailureCount) { breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.ErrorTimeout) * time.Second)) } + if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { + breaker.toHalfOpen() + } + circuitBreakerMap[breaker.Key] = breaker } @@ -256,12 +255,7 @@ func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, brea } // Update the state - err := cb.redis.MSet(ctx, breakerStringsMap).Err() - if err != nil { - return err - } - - return nil + return cb.redis.MSet(ctx, breakerStringsMap).Err() } func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { @@ -275,7 +269,7 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir return nil, err } - var circuitBreakers []CircuitBreaker + circuitBreakers := make([]CircuitBreaker, len(res)) for i := range res { c := CircuitBreaker{} asBytes := []byte(res[i].(string)) @@ -284,7 +278,7 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir return nil, innerErr } - circuitBreakers = append(circuitBreakers, c) + circuitBreakers[i] = c } return circuitBreakers, nil diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index d583bf3db5..28faf4ef15 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -42,7 +42,7 @@ func TestNewCircuitBreaker(t *testing.T) { err = re.Del(ctx, keys...).Err() require.NoError(t, err) - testClock := clock.NewSimulatedClock(time.Now()) + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) c := &CircuitBreakerConfig{ SampleRate: 2, @@ -76,9 +76,6 @@ func TestNewCircuitBreaker(t *testing.T) { { pollResult(t, endpointId, 1, 4), }, - { - pollResult(t, endpointId, 0, 5), - }, } for i := 0; i < len(pollResults); i++ { From 1081b3c90a8cc5f0819c986529646d574b88d853 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Thu, 22 Aug 2024 14:51:04 +0200 Subject: [PATCH 09/48] feat: add index to migration --- sql/1724236900.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/1724236900.sql b/sql/1724236900.sql index 6bd47e43e7..7aa0a7e44e 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -7,6 +7,7 @@ alter table convoy.configurations add column if not exists cb_success_threshold alter table convoy.configurations add column if not exists cb_observability_window int not null default 5; alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[5, 10]; alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 5; +create index if not exists idx_delivery_attempts_created_at ON convoy.delivery_attempts (created_at); -- +migrate Down alter table convoy.configurations drop column if exists cb_sample_rate; From 9dba34322927a4a35f2c3e30e75ecad8011a1ae7 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Thu, 22 Aug 2024 14:56:41 +0200 Subject: [PATCH 10/48] feat: personal review first pass; add func to validate custom config; fixed potential panic when decoding payload from redis; fixed potential divide by zero error; delete stale keys after a while; added a func to get breaker error; added exit case in main-loop; set expiry on all breaker keys; --- cmd/worker/worker.go | 14 +- config/config.go | 18 +- database/postgres/delivery_attempts.go | 2 +- datastore/models.go | 16 +- datastore/repository.go | 2 +- mocks/repository.go | 2 +- pkg/circuit_breaker/circuit_breaker.go | 314 +++++++++++++++----- pkg/circuit_breaker/circuit_breaker_test.go | 87 +++++- 8 files changed, 353 insertions(+), 102 deletions(-) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index cbd01d56d2..02b75b808f 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -243,8 +243,18 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - circuitBreakerManager := circuit_breaker.NewCircuitBreakerManager(rd.Client()).WithConfig(configuration.ToCircuitBreakerConfig()) - go circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) + circuitBreakerManager, err := circuit_breaker.NewCircuitBreakerManager(rd.Client()).WithConfig(configuration.ToCircuitBreakerConfig()) + if err != nil { + a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") + } + + go func() { + innerErr := circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) + if innerErr != nil { + // todo(raymond): should this be Fatal? + a.Logger.WithError(innerErr).Fatal("circuit breaker manager failed") + } + }() consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, diff --git a/config/config.go b/config/config.go index ceb32b5ffa..6f2ebebe18 100644 --- a/config/config.go +++ b/config/config.go @@ -81,7 +81,7 @@ var DefaultConfiguration = Configuration{ FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: []int{5, 10}, + NotificationThresholds: []uint64{5, 10}, ConsecutiveFailureThreshold: 10, }, Auth: AuthConfiguration{ @@ -272,14 +272,14 @@ type RetentionPolicyConfiguration struct { } type CircuitBreakerConfiguration struct { - SampleRate int `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` - FailureCount int `json:"failure_count" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_COUNT"` - ErrorTimeout int `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` - FailureThreshold float64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` - SuccessThreshold int `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` - ObservabilityWindow int `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` - NotificationThresholds []int `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` - ConsecutiveFailureThreshold int `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` + SampleRate uint64 `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` + FailureCount uint64 `json:"failure_count" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_COUNT"` + ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` + FailureThreshold float64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` + SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` + ObservabilityWindow uint64 `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` + NotificationThresholds []uint64 `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` } type AnalyticsConfiguration struct { diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index cd39a7f03d..0fd219878c 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -131,7 +131,7 @@ func (d *deliveryAttemptRepo) DeleteProjectDeliveriesAttempts(ctx context.Contex return nil } -func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []circuit_breaker.PollResult, err error) { +func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64) (results []circuit_breaker.PollResult, err error) { query := ` SELECT endpoint_id AS key, diff --git a/datastore/models.go b/datastore/models.go index a3ea4378d5..a4e8776b43 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -1345,9 +1345,9 @@ func (c *Configuration) GetCircuitBreakerConfig() CircuitBreakerConfig { } func (c *Configuration) ToCircuitBreakerConfig() *circuit_breaker.CircuitBreakerConfig { - notificationThresholds := make([]int, len(c.CircuitBreakerConfig.NotificationThresholds)) + notificationThresholds := make([]uint64, len(c.CircuitBreakerConfig.NotificationThresholds)) for i := range c.CircuitBreakerConfig.NotificationThresholds { - notificationThresholds[i] = int(c.CircuitBreakerConfig.NotificationThresholds[i]) + notificationThresholds[i] = uint64(c.CircuitBreakerConfig.NotificationThresholds[i]) } return &circuit_breaker.CircuitBreakerConfig{ @@ -1390,14 +1390,14 @@ type OnPremStorage struct { } type CircuitBreakerConfig struct { - SampleRate int `json:"sample_rate" db:"sample_rate"` - ErrorTimeout int `json:"error_timeout" db:"error_timeout"` + SampleRate uint64 `json:"sample_rate" db:"sample_rate"` + ErrorTimeout uint64 `json:"error_timeout" db:"error_timeout"` FailureThreshold float64 `json:"failure_threshold" db:"failure_threshold"` - FailureCount int `json:"failure_count" db:"failure_count"` - SuccessThreshold int `json:"success_threshold" db:"success_threshold"` - ObservabilityWindow int `json:"observability_window" db:"observability_window"` + FailureCount uint64 `json:"failure_count" db:"failure_count"` + SuccessThreshold uint64 `json:"success_threshold" db:"success_threshold"` + ObservabilityWindow uint64 `json:"observability_window" db:"observability_window"` NotificationThresholds pq.Int64Array `json:"notification_thresholds" db:"notification_thresholds"` - ConsecutiveFailureThreshold int `json:"consecutive_failure_threshold" db:"consecutive_failure_threshold"` + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" db:"consecutive_failure_threshold"` } type OrganisationMember struct { diff --git a/datastore/repository.go b/datastore/repository.go index 5f6bfd47ff..bfdc296528 100644 --- a/datastore/repository.go +++ b/datastore/repository.go @@ -204,5 +204,5 @@ type DeliveryAttemptsRepository interface { FindDeliveryAttemptById(context.Context, string, string) (*DeliveryAttempt, error) FindDeliveryAttempts(context.Context, string) ([]DeliveryAttempt, error) DeleteProjectDeliveriesAttempts(ctx context.Context, projectID string, filter *DeliveryAttemptsFilter, hardDelete bool) error - GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) (results []circuit_breaker.PollResult, err error) + GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64) (results []circuit_breaker.PollResult, err error) } diff --git a/mocks/repository.go b/mocks/repository.go index 86436148e2..55c80ffea3 100644 --- a/mocks/repository.go +++ b/mocks/repository.go @@ -2554,7 +2554,7 @@ func (mr *MockDeliveryAttemptsRepositoryMockRecorder) FindDeliveryAttempts(arg0, } // GetFailureAndSuccessCounts mocks base method. -func (m *MockDeliveryAttemptsRepository) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration int) ([]circuit_breaker.PollResult, error) { +func (m *MockDeliveryAttemptsRepository) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64) ([]circuit_breaker.PollResult, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetFailureAndSuccessCounts", ctx, lookBackDuration) ret0, _ := ret[0].([]circuit_breaker.PollResult) diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 75b76dcb50..f5443bde01 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -11,68 +11,120 @@ import ( "time" ) -const prefix = "breaker" +const ( + prefix = "breaker:" + keyTTL = time.Hour +) + +type PollFunc func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) var ( - // ErrTooManyRequests is returned when the CB state is half open and the requests count is over the cb maxRequests - ErrTooManyRequests = errors.New("too many requests") + // ErrTooManyRequests is returned when the CB state is half open and the request count is over the failureThreshold + ErrTooManyRequests = errors.New("[circuit breaker] too many requests") // ErrOpenState is returned when the CB state is open - ErrOpenState = errors.New("circuit breaker is open") + ErrOpenState = errors.New("[circuit breaker] circuit breaker is open") - // ErrCircuitBreakerNotFound is returned when the circuit breaker is not found - ErrCircuitBreakerNotFound = errors.New("circuit breaker not found") + // ErrCircuitBreakerNotFound is returned when the CB is not found + ErrCircuitBreakerNotFound = errors.New("[circuit breaker] circuit breaker not found") ) // CircuitBreakerConfig is the configuration that all the circuit breakers will use type CircuitBreakerConfig struct { // SampleRate is the time interval (in seconds) at which the data source // is polled to determine the number successful and failed requests - SampleRate int `json:"sample_rate"` + SampleRate uint64 `json:"sample_rate"` // ErrorTimeout is the time (in seconds) after which a circuit breaker goes // into the half-open state from the open state - ErrorTimeout int `json:"error_timeout"` + ErrorTimeout uint64 `json:"error_timeout"` // FailureThreshold is the % of failed requests in the observability window // after which the breaker will go into the open state FailureThreshold float64 `json:"failure_threshold"` // FailureCount total number of failed requests in the observability window - FailureCount int `json:"failure_count"` + FailureCount uint64 `json:"failure_count"` // SuccessThreshold is the % of successful requests in the observability window // after which a circuit breaker in the half-open state will go into the closed state - SuccessThreshold int `json:"success_threshold"` + SuccessThreshold uint64 `json:"success_threshold"` // ObservabilityWindow is how far back in time (in minutes) the data source is // polled when determining the number successful and failed requests - ObservabilityWindow int `json:"observability_window"` + ObservabilityWindow uint64 `json:"observability_window"` // NotificationThresholds These are the error counts after which we will send out notifications. - NotificationThresholds []int `json:"notification_thresholds"` + NotificationThresholds []uint64 `json:"notification_thresholds"` // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. // E.g., after 10 consecutive transitions from half-open → open we should disable it. - ConsecutiveFailureThreshold int `json:"consecutive_failure_threshold"` + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold"` } -// State is a type that represents a state of CircuitBreaker. -type State int +func (c *CircuitBreakerConfig) Validate() error { + var errs []string -func stateFromString(s string) State { - switch s { - case "open": - return StateOpen - case "closed": - return StateClosed - case "half-open": - return StateHalfOpen - } - return StateClosed + if c.SampleRate == 0 { + errs = append(errs, "SampleRate must be greater than 0") + } + + if c.ErrorTimeout == 0 { + errs = append(errs, "ErrorTimeout must be greater than 0") + } + + if c.FailureThreshold < 0 || c.FailureThreshold > 1 { + errs = append(errs, "FailureThreshold must be between 0 and 1") + } + + if c.FailureCount == 0 { + errs = append(errs, "FailureCount must be greater than 0") + } + + if c.SuccessThreshold == 0 { + errs = append(errs, "SuccessThreshold must be greater than 0") + } + + if c.ObservabilityWindow == 0 { + errs = append(errs, "ObservabilityWindow must be greater than 0") + } + + if len(c.NotificationThresholds) == 0 { + errs = append(errs, "NotificationThresholds must contain at least one threshold") + } else { + for i, threshold := range c.NotificationThresholds { + if threshold == 0 { + errs = append(errs, fmt.Sprintf("Notification thresholds at index [%d] = %d must be greater than 0", i, threshold)) + } + } + } + + if c.ConsecutiveFailureThreshold == 0 { + errs = append(errs, "ConsecutiveFailureThreshold must be greater than 0") + } + + if len(errs) > 0 { + return errors.New("CircuitBreakerConfig validation failed: " + joinErrors(errs)) + } + + return nil +} + +func joinErrors(errs []string) string { + result := "" + for i, err := range errs { + if i > 0 { + result += "; " + } + result += err + } + return result } -// These constants are states of the CircuitBreaker. +// State represents a state of a CircuitBreaker. +type State int + +// These are the states of a CircuitBreaker. const ( StateClosed State = iota StateHalfOpen @@ -96,13 +148,13 @@ func (s State) String() string { type CircuitBreaker struct { Key string `json:"key"` State State `json:"state"` - Requests int `json:"requests"` - WillResetAt time.Time `json:"will_reset_at"` - TotalFailures int `json:"total_failures"` - TotalSuccesses int `json:"total_successes"` - ConsecutiveFailures int `json:"consecutive_failures"` - ConsecutiveSuccesses int `json:"consecutive_successes"` + Requests uint64 `json:"requests"` FailureRate float64 `json:"failure_rate"` + WillResetAt time.Time `json:"will_reset_at"` + TotalFailures uint64 `json:"total_failures"` + TotalSuccesses uint64 `json:"total_successes"` + ConsecutiveFailures uint64 `json:"consecutive_failures"` + ConsecutiveSuccesses uint64 `json:"consecutive_successes"` } func (b *CircuitBreaker) String() (s string, err error) { @@ -132,15 +184,14 @@ func (b *CircuitBreaker) resetCircuitBreaker() { type PollResult struct { Key string `json:"key" db:"key"` - Failures int `json:"failures" db:"failures"` - Successes int `json:"successes" db:"successes"` + Failures uint64 `json:"failures" db:"failures"` + Successes uint64 `json:"successes" db:"successes"` } type CircuitBreakerManager struct { - config *CircuitBreakerConfig - breakers []CircuitBreaker - clock clock.Clock - redis *redis.Client + config *CircuitBreakerConfig + clock clock.Clock + redis *redis.Client } func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManager { @@ -151,7 +202,7 @@ func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManag FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: []int{5, 10}, + NotificationThresholds: []uint64{5, 10}, ConsecutiveFailureThreshold: 10, } @@ -164,14 +215,26 @@ func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManag return r } -func (cb *CircuitBreakerManager) WithClock(c clock.Clock) *CircuitBreakerManager { +func (cb *CircuitBreakerManager) WithClock(c clock.Clock) (*CircuitBreakerManager, error) { + if cb.clock == nil { + return nil, errors.New("clock must not be nil") + } + cb.clock = c - return cb + return cb, nil } -func (cb *CircuitBreakerManager) WithConfig(config *CircuitBreakerConfig) *CircuitBreakerManager { +func (cb *CircuitBreakerManager) WithConfig(config *CircuitBreakerConfig) (*CircuitBreakerManager, error) { + if config == nil { + return nil, errors.New("config must not be nil") + } + cb.config = config - return cb + + if err := config.Validate(); err != nil { + return nil, err + } + return cb, nil } func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { @@ -183,29 +246,44 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] } res, err := cb.redis.MGet(context.Background(), keys...).Result() - if err != nil && !errors.Is(err, redis.Nil) { + if err != nil { + if errors.Is(err, redis.Nil) { + return nil + } return err } - var circuitBreakers []CircuitBreaker + circuitBreakers := make([]CircuitBreaker, len(pollResults)) for i := range res { if res[i] == nil { c := CircuitBreaker{ State: StateClosed, Key: pollResults[i].Key, } - circuitBreakers = append(circuitBreakers, c) + circuitBreakers[i] = c continue } c := CircuitBreaker{} - asBytes := []byte(res[i].(string)) + str, ok := res[i].(string) + if !ok { + log.Errorf("[circuit breaker] breaker with key (%s) is corrupted", keys[i]) + + // the circuit breaker is corrupted, create a new one in its place + circuitBreakers[i] = CircuitBreaker{ + State: StateClosed, + Key: keys[i], + } + continue + } + + asBytes := []byte(str) innerErr := msgpack.DecodeMsgPack(asBytes, &c) if innerErr != nil { return innerErr } - circuitBreakers = append(circuitBreakers, c) + circuitBreakers[i] = c } resultsMap := make(map[string]PollResult) @@ -218,10 +296,15 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] for _, breaker := range circuitBreakers { result := resultsMap[breaker.Key] - breaker.Requests = result.Successes + result.Failures breaker.TotalFailures = result.Failures breaker.TotalSuccesses = result.Successes - breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.TotalSuccesses+breaker.TotalFailures) + breaker.Requests = breaker.TotalSuccesses + breaker.TotalFailures + + if breaker.Requests == 0 { + breaker.FailureRate = 0 + } else { + breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.Requests) + } if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { breaker.resetCircuitBreaker() @@ -237,7 +320,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] } if err = cb.updateCircuitBreakers(ctx, circuitBreakerMap); err != nil { - log.WithError(err).Error("failed to update state") + log.WithError(err).Error("[circuit breaker] failed to update state") return err } @@ -255,7 +338,25 @@ func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, brea } // Update the state - return cb.redis.MSet(ctx, breakerStringsMap).Err() + err := cb.redis.MSet(ctx, breakerStringsMap).Err() + if err != nil { + return err + } + + pipe := cb.redis.TxPipeline() + for key := range breakers { + err = pipe.Expire(ctx, key, keyTTL).Err() + if err != nil { + return err + } + } + + _, err = pipe.Exec(ctx) + if err != nil { + return err + } + + return nil } func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { @@ -265,7 +366,10 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir } res, err := cb.redis.MGet(ctx, keys...).Result() - if err != nil && !errors.Is(err, redis.Nil) { + if err != nil { + if errors.Is(err, redis.Nil) { + return nil, nil + } return nil, err } @@ -284,19 +388,37 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir return circuitBreakers, nil } +func (cb *CircuitBreakerManager) GetCircuitBreakerError(ctx context.Context, key string) error { + b, err := cb.GetCircuitBreaker(ctx, key) + if err != nil { + return err + } + + switch b.State { + case StateOpen: + return ErrOpenState + case StateHalfOpen: + if b.TotalFailures > cb.config.FailureCount { + return ErrTooManyRequests + } + return nil + default: + return nil + } +} + // GetCircuitBreaker is used to get fetch the circuit breaker state before executing a function func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - breakerKey := fmt.Sprintf("%s:%s", prefix, key) + breakerKey := fmt.Sprintf("%s%s", prefix, key) res, err := cb.redis.Get(ctx, breakerKey).Result() - if err != nil && !errors.Is(err, redis.Nil) { + if err != nil { + if errors.Is(err, redis.Nil) { + return nil, ErrCircuitBreakerNotFound + } return nil, err } - if err != nil && errors.Is(err, redis.Nil) { - return nil, ErrCircuitBreakerNotFound - } - err = msgpack.DecodeMsgPack([]byte(res), &c) if err != nil { return nil, err @@ -305,26 +427,64 @@ func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key stri return c, nil } -func (cb *CircuitBreakerManager) Start(ctx context.Context, poolFunc func(ctx context.Context, lookBackDuration int) (results []PollResult, err error)) { - for { - // Get the failure and success counts from the last X minutes - pollResults, err := poolFunc(ctx, cb.config.ObservabilityWindow) - if err != nil { - log.WithError(err).Error("poll db failed") - time.Sleep(time.Duration(cb.config.SampleRate) * time.Second) - continue - } +func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { + // Get the failure and success counts from the last X minutes + pollResults, err := pollFunc(ctx, cb.config.ObservabilityWindow) + if err != nil { + return fmt.Errorf("poll function failed: %w", err) + } - if len(pollResults) == 0 { - // there's nothing to update - time.Sleep(time.Duration(cb.config.SampleRate) * time.Second) - continue - } + if len(pollResults) == 0 { + return nil // Nothing to update + } - err = cb.sampleStore(ctx, pollResults) - if err != nil { - log.WithError(err).Error("Failed to sample events and update state") + if err = cb.sampleStore(ctx, pollResults); err != nil { + return fmt.Errorf("[circuit breaker] failed to sample events and update state: %w", err) + } + + return nil +} + +func (cb *CircuitBreakerManager) cleanup(ctx context.Context) error { + keys, err := cb.redis.Keys(ctx, fmt.Sprintf("%s%s", prefix, "*")).Result() + if err != nil { + return fmt.Errorf("failed to get keys for cleanup: %w", err) + } + + pipe := cb.redis.TxPipeline() + for _, key := range keys { + pipe.Del(ctx, key) + } + + _, err = pipe.Exec(ctx) + if err != nil { + return fmt.Errorf("failed to execute cleanup pipeline: %w", err) + } + + return nil +} + +func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) error { + ticker := time.NewTicker(time.Duration(cb.config.SampleRate) * time.Second) + defer ticker.Stop() + + // Run cleanup daily + // todo(raymond): should this be run by asynq? + cleanupTicker := time.NewTicker(24 * time.Hour) + defer cleanupTicker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { + log.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") + } + case <-cleanupTicker.C: + if err := cb.cleanup(ctx); err != nil { + log.WithError(err).Error("[circuit breaker] failed to cleanup circuit breakers") + } } - time.Sleep(time.Duration(cb.config.SampleRate) * time.Second) } } diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index 28faf4ef15..4aca4d3c88 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -20,7 +20,7 @@ func getRedis(t *testing.T) (client redis.UniversalClient, err error) { return redis.NewClient(opts), nil } -func pollResult(t *testing.T, key string, failureCount, successCount int) PollResult { +func pollResult(t *testing.T, key string, failureCount, successCount uint64) PollResult { t.Helper() return PollResult{ @@ -51,10 +51,15 @@ func TestNewCircuitBreaker(t *testing.T) { FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, - NotificationThresholds: []int{10}, + NotificationThresholds: []uint64{10}, ConsecutiveFailureThreshold: 10, } - b := NewCircuitBreakerManager(re).WithClock(testClock).WithConfig(c) + + b, err := NewCircuitBreakerManager(re).WithClock(testClock) + require.NoError(t, err) + + b, err = b.WithConfig(c) + require.NoError(t, err) endpointId := "endpoint-1" pollResults := [][]PollResult{ @@ -94,3 +99,79 @@ func TestNewCircuitBreaker(t *testing.T) { require.Equal(t, breaker.State, StateClosed) } + +func TestNewCircuitBreaker_AddNewBreakerMidway(t *testing.T) { + ctx := context.Background() + + re, err := getRedis(t) + require.NoError(t, err) + + keys, err := re.Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + err = re.Del(ctx, keys...).Err() + require.NoError(t, err) + + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + + c := &CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 3, + SuccessThreshold: 1, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10}, + ConsecutiveFailureThreshold: 10, + } + b, err := NewCircuitBreakerManager(re).WithClock(testClock) + require.NoError(t, err) + + b, err = b.WithConfig(c) + require.NoError(t, err) + + endpoint1 := "endpoint-1" + endpoint2 := "endpoint-2" + pollResults := [][]PollResult{ + { + pollResult(t, endpoint1, 1, 0), + }, + { + pollResult(t, endpoint1, 2, 0), + }, + { + pollResult(t, endpoint1, 2, 1), + pollResult(t, endpoint2, 1, 0), + }, + { + pollResult(t, endpoint1, 2, 2), + pollResult(t, endpoint2, 1, 1), + }, + { + pollResult(t, endpoint1, 2, 3), + pollResult(t, endpoint2, 0, 2), + }, + { + pollResult(t, endpoint1, 1, 4), + pollResult(t, endpoint2, 1, 1), + }, + } + + for i := 0; i < len(pollResults); i++ { + err = b.sampleStore(ctx, pollResults[i]) + require.NoError(t, err) + + if i > 1 { + breaker, innerErr := b.GetCircuitBreaker(ctx, endpoint2) + require.NoError(t, innerErr) + t.Logf("%+v\n", breaker) + } + + testClock.AdvanceTime(time.Minute) + } + + breakers, innerErr := b.loadCircuitBreakers(ctx) + require.NoError(t, innerErr) + + require.Len(t, breakers, 2) +} From 848559ed795a98500a438c2b32be5b84e202a1fb Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 23 Aug 2024 14:30:08 +0200 Subject: [PATCH 11/48] feat: personal review second pass; added CircuitBreakerManager.CanExecute(); set the redis key ttl to the config observability window length; added the circuit breaker to the event dispatch handlers; excluded circuit breaker errors from asynq types --- cmd/worker/worker.go | 10 +-- pkg/circuit_breaker/circuit_breaker.go | 88 +++++++++++++++------ pkg/circuit_breaker/circuit_breaker_test.go | 10 +-- worker/consumer.go | 5 ++ worker/task/process_event_delivery.go | 9 ++- worker/task/process_retry_event_delivery.go | 9 ++- worker/task/task.go | 19 +++++ 7 files changed, 113 insertions(+), 37 deletions(-) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 02b75b808f..4f7fcdb84b 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -248,13 +248,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") } - go func() { - innerErr := circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) - if innerErr != nil { - // todo(raymond): should this be Fatal? - a.Logger.WithError(innerErr).Fatal("circuit breaker manager failed") - } - }() + go circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, @@ -264,6 +258,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte rateLimiter, dispatcher, attemptRepo, + circuitBreakerManager, ), newTelemetry) consumer.RegisterHandlers(convoy.CreateEventProcessor, task.ProcessEventCreation( @@ -283,6 +278,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte rateLimiter, dispatcher, attemptRepo, + circuitBreakerManager, ), newTelemetry) consumer.RegisterHandlers(convoy.CreateBroadcastEventProcessor, task.ProcessBroadcastEventCreation( diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index f5443bde01..e7bbb30dab 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -11,22 +11,25 @@ import ( "time" ) -const ( - prefix = "breaker:" - keyTTL = time.Hour -) +const prefix = "breaker:" type PollFunc func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) var ( - // ErrTooManyRequests is returned when the CB state is half open and the request count is over the failureThreshold + // ErrTooManyRequests is returned when the circuit breaker state is half open and the request count is over the failureThreshold ErrTooManyRequests = errors.New("[circuit breaker] too many requests") - // ErrOpenState is returned when the CB state is open + // ErrOpenState is returned when the circuit breaker state is open ErrOpenState = errors.New("[circuit breaker] circuit breaker is open") - // ErrCircuitBreakerNotFound is returned when the CB is not found + // ErrCircuitBreakerNotFound is returned when the circuit breaker is not found ErrCircuitBreakerNotFound = errors.New("[circuit breaker] circuit breaker not found") + + // ErrClockMustNotBeNil is returned when a nil clock is passed to NewCircuitBreakerManager + ErrClockMustNotBeNil = errors.New("[circuit breaker] clock must not be nil") + + // ErrConfigMustNotBeNil is returned when a nil config is passed to NewCircuitBreakerManager + ErrConfigMustNotBeNil = errors.New("[circuit breaker] config must not be nil") ) // CircuitBreakerConfig is the configuration that all the circuit breakers will use @@ -104,7 +107,7 @@ func (c *CircuitBreakerConfig) Validate() error { } if len(errs) > 0 { - return errors.New("CircuitBreakerConfig validation failed: " + joinErrors(errs)) + return fmt.Errorf("config validation failed with errors: %s", joinErrors(errs)) } return nil @@ -217,7 +220,7 @@ func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManag func (cb *CircuitBreakerManager) WithClock(c clock.Clock) (*CircuitBreakerManager, error) { if cb.clock == nil { - return nil, errors.New("clock must not be nil") + return nil, ErrClockMustNotBeNil } cb.clock = c @@ -226,7 +229,7 @@ func (cb *CircuitBreakerManager) WithClock(c clock.Clock) (*CircuitBreakerManage func (cb *CircuitBreakerManager) WithConfig(config *CircuitBreakerConfig) (*CircuitBreakerManager, error) { if config == nil { - return nil, errors.New("config must not be nil") + return nil, ErrConfigMustNotBeNil } cb.config = config @@ -240,7 +243,7 @@ func (cb *CircuitBreakerManager) WithConfig(config *CircuitBreakerConfig) (*Circ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { var keys []string for i := range pollResults { - key := fmt.Sprintf("%s:%s", prefix, pollResults[i].Key) + key := fmt.Sprintf("%s%s", prefix, pollResults[i].Key) keys = append(keys, key) pollResults[i].Key = key } @@ -267,7 +270,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] c := CircuitBreaker{} str, ok := res[i].(string) if !ok { - log.Errorf("[circuit breaker] breaker with key (%s) is corrupted", keys[i]) + log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) // the circuit breaker is corrupted, create a new one in its place circuitBreakers[i] = CircuitBreaker{ @@ -345,7 +348,7 @@ func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, brea pipe := cb.redis.TxPipeline() for key := range breakers { - err = pipe.Expire(ctx, key, keyTTL).Err() + err = pipe.Expire(ctx, key, time.Duration(cb.config.ObservabilityWindow)*time.Minute).Err() if err != nil { return err } @@ -388,12 +391,7 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir return circuitBreakers, nil } -func (cb *CircuitBreakerManager) GetCircuitBreakerError(ctx context.Context, key string) error { - b, err := cb.GetCircuitBreaker(ctx, key) - if err != nil { - return err - } - +func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error { switch b.State { case StateOpen: return ErrOpenState @@ -407,7 +405,53 @@ func (cb *CircuitBreakerManager) GetCircuitBreakerError(ctx context.Context, key } } -// GetCircuitBreaker is used to get fetch the circuit breaker state before executing a function +// CanExecute checks if the circuit breaker for a key will return an error for the current state. +// It will not return an error if it is in the closed state or half-open state when the failure +// threshold has not been reached, it will fail-open if the circuit breaker is not found +func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) error { + breaker, err := cb.getCircuitBreaker(ctx, key) + if err != nil { + return err + } + + if breaker != nil { + switch breaker.State { + case StateOpen, StateHalfOpen: + return cb.getCircuitBreakerError(*breaker) + default: + return nil + } + } + + return nil +} + +// getCircuitBreaker is used to get fetch the circuit breaker state, +// it fails open if the circuit breaker for that key is not found +func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { + breakerKey := fmt.Sprintf("%s%s", prefix, key) + + res, err := cb.redis.Get(ctx, breakerKey).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + // a circuit breaker was not found for this key; + // it probably hasn't been created; + // we should fail open + return nil, nil + } + return nil, err + } + + err = msgpack.DecodeMsgPack([]byte(res), &c) + if err != nil { + return nil, err + } + + return c, nil +} + +// GetCircuitBreaker is used to get fetch the circuit breaker state, +// it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { breakerKey := fmt.Sprintf("%s%s", prefix, key) @@ -464,7 +508,7 @@ func (cb *CircuitBreakerManager) cleanup(ctx context.Context) error { return nil } -func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) error { +func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { ticker := time.NewTicker(time.Duration(cb.config.SampleRate) * time.Second) defer ticker.Stop() @@ -476,7 +520,7 @@ func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) e for { select { case <-ctx.Done(): - return ctx.Err() + return case <-ticker.C: if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { log.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index 4aca4d3c88..b3553bd80f 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -39,8 +39,10 @@ func TestNewCircuitBreaker(t *testing.T) { keys, err := re.Keys(ctx, "breaker*").Result() require.NoError(t, err) - err = re.Del(ctx, keys...).Err() - require.NoError(t, err) + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) @@ -87,10 +89,6 @@ func TestNewCircuitBreaker(t *testing.T) { innerErr := b.sampleStore(ctx, pollResults[i]) require.NoError(t, innerErr) - breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) - require.NoError(t, innerErr) - t.Logf("%+v\n", breaker) - testClock.AdvanceTime(time.Minute) } diff --git a/worker/consumer.go b/worker/consumer.go index 3ade5fe64f..b17cb7474d 100644 --- a/worker/consumer.go +++ b/worker/consumer.go @@ -44,6 +44,11 @@ func NewConsumer(ctx context.Context, consumerPoolSize int, q queue.Queuer, lo l if _, ok := err.(*task.RateLimitError); ok { return false } + + if _, ok := err.(*task.CircuitBreakerError); ok { + return false + } + return true }, RetryDelayFunc: task.GetRetryDelay, diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index 3e5eff3913..730925e46e 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/frain-dev/convoy/internal/pkg/metrics" + "github.com/frain-dev/convoy/pkg/circuit_breaker" "time" "github.com/frain-dev/convoy/internal/pkg/limiter" @@ -29,7 +30,8 @@ import ( ) func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, - projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, attemptsRepo datastore.DeliveryAttemptsRepository, + projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, + attemptsRepo datastore.DeliveryAttemptsRepository, manager *circuit_breaker.CircuitBreakerManager, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) (err error) { var data EventDelivery @@ -111,6 +113,11 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive return &RateLimitError{Err: ErrRateLimit, delay: time.Duration(endpoint.RateLimitDuration) * time.Second} } + err = manager.CanExecute(ctx, endpoint.UID) + if err != nil { + return &CircuitBreakerError{Err: err} + } + err = eventDeliveryRepo.UpdateStatusOfEventDelivery(ctx, project.UID, *eventDelivery, datastore.ProcessingEventStatus) if err != nil { return &DeliveryError{Err: err} diff --git a/worker/task/process_retry_event_delivery.go b/worker/task/process_retry_event_delivery.go index bf90c28abe..1dd8b76827 100644 --- a/worker/task/process_retry_event_delivery.go +++ b/worker/task/process_retry_event_delivery.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/frain-dev/convoy/pkg/circuit_breaker" "time" "github.com/frain-dev/convoy/internal/pkg/limiter" @@ -38,7 +39,8 @@ var ( ) func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, - projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, attemptsRepo datastore.DeliveryAttemptsRepository, + projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, + attemptsRepo datastore.DeliveryAttemptsRepository, manager *circuit_breaker.CircuitBreakerManager, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var data EventDelivery @@ -98,6 +100,11 @@ func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventD return &RateLimitError{Err: ErrRateLimit, delay: time.Duration(endpoint.RateLimitDuration) * time.Second} } + err = manager.CanExecute(ctx, endpoint.UID) + if err != nil { + return &DeliveryError{Err: err} + } + err = eventDeliveryRepo.UpdateStatusOfEventDelivery(ctx, project.UID, *eventDelivery, datastore.ProcessingEventStatus) if err != nil { return &EndpointError{Err: err, delay: defaultEventDelay} diff --git a/worker/task/task.go b/worker/task/task.go index e6000e7dcf..ce391ce8e1 100644 --- a/worker/task/task.go +++ b/worker/task/task.go @@ -51,6 +51,9 @@ func GetRetryDelay(n int, err error, t *asynq.Task) time.Duration { if rateLimitError, ok := err.(*RateLimitError); ok { return rateLimitError.Delay() } + if circuitBreakerError, ok := err.(*CircuitBreakerError); ok { + return circuitBreakerError.Delay() + } return asynq.DefaultRetryDelayFunc(n, err, t) } @@ -105,3 +108,19 @@ func (ec *EventDeliveryConfig) RateLimitConfig() *RateLimitConfig { return rlc } + +type CircuitBreakerError struct { + delay time.Duration + Err error +} + +func (e *CircuitBreakerError) Error() string { + if e.Err != nil { + return e.Err.Error() + } + return "" +} + +func (e *CircuitBreakerError) Delay() time.Duration { + return e.delay +} From 2880b8ccbc5a72a002cd400b456b21c406c05d45 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 23 Aug 2024 16:17:16 +0200 Subject: [PATCH 12/48] feat: personal review third pass; use strings builder to build config error; add more config validation rules; prefer redis.Set over redis.MSet; remove cleanup job --- pkg/circuit_breaker/circuit_breaker.go | 130 ++++++++++--------------- 1 file changed, 51 insertions(+), 79 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index e7bbb30dab..1950c08cba 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -8,6 +8,7 @@ import ( "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/pkg/msgpack" "github.com/redis/go-redis/v9" + "strings" "time" ) @@ -66,64 +67,74 @@ type CircuitBreakerConfig struct { } func (c *CircuitBreakerConfig) Validate() error { - var errs []string + var errs strings.Builder if c.SampleRate == 0 { - errs = append(errs, "SampleRate must be greater than 0") + errs.WriteString("SampleRate must be greater than 0") + errs.WriteString("; ") } if c.ErrorTimeout == 0 { - errs = append(errs, "ErrorTimeout must be greater than 0") + errs.WriteString("ErrorTimeout must be greater than 0") + errs.WriteString("; ") } if c.FailureThreshold < 0 || c.FailureThreshold > 1 { - errs = append(errs, "FailureThreshold must be between 0 and 1") + errs.WriteString("FailureThreshold must be between 0 and 1") + errs.WriteString("; ") } if c.FailureCount == 0 { - errs = append(errs, "FailureCount must be greater than 0") + errs.WriteString("FailureCount must be greater than 0") + errs.WriteString("; ") } if c.SuccessThreshold == 0 { - errs = append(errs, "SuccessThreshold must be greater than 0") + errs.WriteString("SuccessThreshold must be greater than 0") + errs.WriteString("; ") } if c.ObservabilityWindow == 0 { - errs = append(errs, "ObservabilityWindow must be greater than 0") + errs.WriteString("ObservabilityWindow must be greater than 0") + errs.WriteString("; ") + } + + if c.ObservabilityWindow <= c.SampleRate { + errs.WriteString("ObservabilityWindow must be greater than the SampleRate") + errs.WriteString("; ") } if len(c.NotificationThresholds) == 0 { - errs = append(errs, "NotificationThresholds must contain at least one threshold") + errs.WriteString("NotificationThresholds must contain at least one threshold") + errs.WriteString("; ") } else { - for i, threshold := range c.NotificationThresholds { - if threshold == 0 { - errs = append(errs, fmt.Sprintf("Notification thresholds at index [%d] = %d must be greater than 0", i, threshold)) + for i := 0; i < len(c.NotificationThresholds); i++ { + if c.NotificationThresholds[i] == 0 { + errs.WriteString(fmt.Sprintf("Notification thresholds at index [%d] = %d must be greater than 0", i, c.NotificationThresholds[i])) + errs.WriteString("; ") + } + } + + for i := 0; i < len(c.NotificationThresholds)-1; i++ { + if c.NotificationThresholds[i] >= c.NotificationThresholds[i+1] { + errs.WriteString("NotificationThresholds should be in ascending order") + errs.WriteString("; ") } } } if c.ConsecutiveFailureThreshold == 0 { - errs = append(errs, "ConsecutiveFailureThreshold must be greater than 0") + errs.WriteString("ConsecutiveFailureThreshold must be greater than 0") + errs.WriteString("; ") } - if len(errs) > 0 { - return fmt.Errorf("config validation failed with errors: %s", joinErrors(errs)) + if errs.Len() > 0 { + return fmt.Errorf("config validation failed with errors: %s", errs.String()) } return nil } -func joinErrors(errs []string) string { - result := "" - for i, err := range errs { - if i > 0 { - result += "; " - } - result += err - } - return result -} - // State represents a state of a CircuitBreaker. type State int @@ -330,27 +341,16 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] return nil } -func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) error { - breakerStringsMap := make(map[string]string, len(breakers)) +func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) (err error) { + pipe := cb.redis.TxPipeline() for key, breaker := range breakers { - val, err := breaker.String() - if err != nil { - return err + val, innerErr := breaker.String() + if innerErr != nil { + return innerErr } - breakerStringsMap[key] = val - } - - // Update the state - err := cb.redis.MSet(ctx, breakerStringsMap).Err() - if err != nil { - return err - } - pipe := cb.redis.TxPipeline() - for key := range breakers { - err = pipe.Expire(ctx, key, time.Duration(cb.config.ObservabilityWindow)*time.Minute).Err() - if err != nil { - return err + if innerErr = pipe.Set(ctx, key, val, time.Duration(cb.config.ObservabilityWindow)*time.Minute).Err(); innerErr != nil { + return innerErr } } @@ -409,15 +409,15 @@ func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error // It will not return an error if it is in the closed state or half-open state when the failure // threshold has not been reached, it will fail-open if the circuit breaker is not found func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) error { - breaker, err := cb.getCircuitBreaker(ctx, key) + b, err := cb.getCircuitBreaker(ctx, key) if err != nil { return err } - if breaker != nil { - switch breaker.State { + if b != nil { + switch b.State { case StateOpen, StateHalfOpen: - return cb.getCircuitBreakerError(*breaker) + return cb.getCircuitBreakerError(*b) default: return nil } @@ -429,9 +429,9 @@ func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) err // getCircuitBreaker is used to get fetch the circuit breaker state, // it fails open if the circuit breaker for that key is not found func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - breakerKey := fmt.Sprintf("%s%s", prefix, key) + bKey := fmt.Sprintf("%s%s", prefix, key) - res, err := cb.redis.Get(ctx, breakerKey).Result() + res, err := cb.redis.Get(ctx, bKey).Result() if err != nil { if errors.Is(err, redis.Nil) { // a circuit breaker was not found for this key; @@ -453,9 +453,9 @@ func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key stri // GetCircuitBreaker is used to get fetch the circuit breaker state, // it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - breakerKey := fmt.Sprintf("%s%s", prefix, key) + bKey := fmt.Sprintf("%s%s", prefix, key) - res, err := cb.redis.Get(ctx, breakerKey).Result() + res, err := cb.redis.Get(ctx, bKey).Result() if err != nil { if errors.Is(err, redis.Nil) { return nil, ErrCircuitBreakerNotFound @@ -489,34 +489,10 @@ func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc P return nil } -func (cb *CircuitBreakerManager) cleanup(ctx context.Context) error { - keys, err := cb.redis.Keys(ctx, fmt.Sprintf("%s%s", prefix, "*")).Result() - if err != nil { - return fmt.Errorf("failed to get keys for cleanup: %w", err) - } - - pipe := cb.redis.TxPipeline() - for _, key := range keys { - pipe.Del(ctx, key) - } - - _, err = pipe.Exec(ctx) - if err != nil { - return fmt.Errorf("failed to execute cleanup pipeline: %w", err) - } - - return nil -} - func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { ticker := time.NewTicker(time.Duration(cb.config.SampleRate) * time.Second) defer ticker.Stop() - // Run cleanup daily - // todo(raymond): should this be run by asynq? - cleanupTicker := time.NewTicker(24 * time.Hour) - defer cleanupTicker.Stop() - for { select { case <-ctx.Done(): @@ -525,10 +501,6 @@ func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { log.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") } - case <-cleanupTicker.C: - if err := cb.cleanup(ctx); err != nil { - log.WithError(err).Error("[circuit breaker] failed to cleanup circuit breakers") - } } } } From e268fa4f97732d0a6ce57adab317911901e41d1d Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 28 Aug 2024 14:48:40 +0200 Subject: [PATCH 13/48] feat: use deadline context for redis operations to prevent deadlock; extract redis dependency into an interface --- pkg/circuit_breaker/circuit_breaker.go | 64 +++++++++++++++++--------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 1950c08cba..449c0246ed 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -160,15 +160,14 @@ func (s State) String() string { // CircuitBreaker represents a circuit breaker type CircuitBreaker struct { - Key string `json:"key"` - State State `json:"state"` - Requests uint64 `json:"requests"` - FailureRate float64 `json:"failure_rate"` - WillResetAt time.Time `json:"will_reset_at"` - TotalFailures uint64 `json:"total_failures"` - TotalSuccesses uint64 `json:"total_successes"` - ConsecutiveFailures uint64 `json:"consecutive_failures"` - ConsecutiveSuccesses uint64 `json:"consecutive_successes"` + Key string `json:"key"` + State State `json:"state"` + Requests uint64 `json:"requests"` + FailureRate float64 `json:"failure_rate"` + WillResetAt time.Time `json:"will_reset_at"` + TotalFailures uint64 `json:"total_failures"` + TotalSuccesses uint64 `json:"total_successes"` + ConsecutiveFailures uint64 `json:"consecutive_failures"` } func (b *CircuitBreaker) String() (s string, err error) { @@ -193,7 +192,6 @@ func (b *CircuitBreaker) toHalfOpen() { func (b *CircuitBreaker) resetCircuitBreaker() { b.State = StateClosed b.ConsecutiveFailures = 0 - b.ConsecutiveSuccesses++ } type PollResult struct { @@ -205,10 +203,19 @@ type PollResult struct { type CircuitBreakerManager struct { config *CircuitBreakerConfig clock clock.Clock - redis *redis.Client + redis RedisClient } -func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManager { +type RedisClient interface { + Keys(ctx context.Context, pattern string) *redis.StringSliceCmd + Get(ctx context.Context, key string) *redis.StringCmd + MGet(ctx context.Context, keys ...string) *redis.SliceCmd + Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd + Scan(ctx context.Context, cursor uint64, match string, count int64) *redis.ScanCmd + TxPipeline() redis.Pipeliner +} + +func NewCircuitBreakerManager(client RedisClient) *CircuitBreakerManager { defaultConfig := &CircuitBreakerConfig{ SampleRate: 30, ErrorTimeout: 30, @@ -223,7 +230,7 @@ func NewCircuitBreakerManager(client redis.UniversalClient) *CircuitBreakerManag r := &CircuitBreakerManager{ config: defaultConfig, clock: clock.NewRealClock(), - redis: client.(*redis.Client), + redis: client, } return r @@ -259,7 +266,10 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] pollResults[i].Key = key } - res, err := cb.redis.MGet(context.Background(), keys...).Result() + deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) + defer cancel() + + res, err := cb.redis.MGet(deadlineCtx, keys...).Result() if err != nil { if errors.Is(err, redis.Nil) { return nil @@ -342,6 +352,9 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] } func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) (err error) { + deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) + defer cancel() + pipe := cb.redis.TxPipeline() for key, breaker := range breakers { val, innerErr := breaker.String() @@ -349,12 +362,12 @@ func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, brea return innerErr } - if innerErr = pipe.Set(ctx, key, val, time.Duration(cb.config.ObservabilityWindow)*time.Minute).Err(); innerErr != nil { + if innerErr = pipe.Set(deadlineCtx, key, val, time.Duration(cb.config.ObservabilityWindow)*time.Minute).Err(); innerErr != nil { return innerErr } } - _, err = pipe.Exec(ctx) + _, err = pipe.Exec(deadlineCtx) if err != nil { return err } @@ -363,12 +376,15 @@ func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, brea } func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { - keys, err := cb.redis.Keys(ctx, "breaker*").Result() + deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) + defer cancel() + + keys, err := cb.redis.Keys(deadlineCtx, "breaker*").Result() if err != nil { return nil, err } - res, err := cb.redis.MGet(ctx, keys...).Result() + res, err := cb.redis.MGet(deadlineCtx, keys...).Result() if err != nil { if errors.Is(err, redis.Nil) { return nil, nil @@ -429,9 +445,11 @@ func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) err // getCircuitBreaker is used to get fetch the circuit breaker state, // it fails open if the circuit breaker for that key is not found func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - bKey := fmt.Sprintf("%s%s", prefix, key) + deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) + defer cancel() - res, err := cb.redis.Get(ctx, bKey).Result() + bKey := fmt.Sprintf("%s%s", prefix, key) + res, err := cb.redis.Get(deadlineCtx, bKey).Result() if err != nil { if errors.Is(err, redis.Nil) { // a circuit breaker was not found for this key; @@ -453,9 +471,11 @@ func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key stri // GetCircuitBreaker is used to get fetch the circuit breaker state, // it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - bKey := fmt.Sprintf("%s%s", prefix, key) + deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) + defer cancel() - res, err := cb.redis.Get(ctx, bKey).Result() + bKey := fmt.Sprintf("%s%s", prefix, key) + res, err := cb.redis.Get(deadlineCtx, bKey).Result() if err != nil { if errors.Is(err, redis.Nil) { return nil, ErrCircuitBreakerNotFound From d9a7dd3c16ff22064cca14ffd14e2e77a2e7e2e4 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 30 Aug 2024 11:10:07 +0200 Subject: [PATCH 14/48] feat: split package into separate files, add tests for each component --- cmd/worker/worker.go | 8 +- go.mod | 7 +- go.sum | 14 +- pkg/circuit_breaker/circuit_breaker.go | 484 ---------- .../circuit_breaker_manager.go | 378 ++++++++ .../circuit_breaker_manager_test.go | 832 ++++++++++++++++++ pkg/circuit_breaker/circuit_breaker_test.go | 198 ++--- pkg/circuit_breaker/config.go | 109 +++ pkg/circuit_breaker/config_test.go | 125 +++ pkg/circuit_breaker/metrics.go | 31 + pkg/circuit_breaker/store.go | 158 ++++ pkg/circuit_breaker/store_test.go | 292 ++++++ 12 files changed, 1991 insertions(+), 645 deletions(-) create mode 100644 pkg/circuit_breaker/circuit_breaker_manager.go create mode 100644 pkg/circuit_breaker/circuit_breaker_manager_test.go create mode 100644 pkg/circuit_breaker/config.go create mode 100644 pkg/circuit_breaker/config_test.go create mode 100644 pkg/circuit_breaker/metrics.go create mode 100644 pkg/circuit_breaker/store.go create mode 100644 pkg/circuit_breaker/store_test.go diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 4f7fcdb84b..ddf0ec38cd 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -16,7 +16,8 @@ import ( "github.com/frain-dev/convoy/internal/pkg/smtp" "github.com/frain-dev/convoy/internal/telemetry" "github.com/frain-dev/convoy/net" - "github.com/frain-dev/convoy/pkg/circuit_breaker" + cb "github.com/frain-dev/convoy/pkg/circuit_breaker" + "github.com/frain-dev/convoy/pkg/clock" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/queue" redisQueue "github.com/frain-dev/convoy/queue/redis" @@ -243,7 +244,10 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - circuitBreakerManager, err := circuit_breaker.NewCircuitBreakerManager(rd.Client()).WithConfig(configuration.ToCircuitBreakerConfig()) + circuitBreakerManager, err := cb.NewCircuitBreakerManager( + cb.ConfigOption(configuration.ToCircuitBreakerConfig()), + cb.StoreOption(cb.NewRedisStore(rd.Client(), clock.NewRealClock())), + cb.ClockOption(clock.NewRealClock())) if err != nil { a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") } diff --git a/go.mod b/go.mod index febf6cfe93..da423abb1e 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( github.com/segmentio/kafka-go v0.4.44 github.com/sirupsen/logrus v1.9.3 github.com/slack-go/slack v0.10.2 - github.com/sony/gobreaker/v2 v2.0.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 @@ -69,7 +68,7 @@ require ( go.opentelemetry.io/otel/sdk v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.22.0 + golang.org/x/crypto v0.23.0 google.golang.org/api v0.128.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/guregu/null.v4 v4.0.0 @@ -225,7 +224,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect - golang.org/x/term v0.19.0 // indirect + golang.org/x/term v0.20.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect @@ -285,7 +284,7 @@ require ( golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 98f7864557..f2651855ec 100644 --- a/go.sum +++ b/go.sum @@ -1792,8 +1792,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/sony/gobreaker/v2 v2.0.0 h1:23AaR4JQ65y4rz8JWMzgXw2gKOykZ/qfqYunll4OwJ4= -github.com/sony/gobreaker/v2 v2.0.0/go.mod h1:8JnRUz80DJ1/ne8M8v7nmTs2713i58nIt4s7XcGe/DI= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY= github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI= @@ -2073,8 +2071,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -2408,8 +2406,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2427,8 +2425,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 449c0246ed..ef88664555 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -1,163 +1,10 @@ package circuit_breaker import ( - "context" - "errors" - "fmt" - "github.com/frain-dev/convoy/pkg/clock" - "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/pkg/msgpack" - "github.com/redis/go-redis/v9" - "strings" "time" ) -const prefix = "breaker:" - -type PollFunc func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) - -var ( - // ErrTooManyRequests is returned when the circuit breaker state is half open and the request count is over the failureThreshold - ErrTooManyRequests = errors.New("[circuit breaker] too many requests") - - // ErrOpenState is returned when the circuit breaker state is open - ErrOpenState = errors.New("[circuit breaker] circuit breaker is open") - - // ErrCircuitBreakerNotFound is returned when the circuit breaker is not found - ErrCircuitBreakerNotFound = errors.New("[circuit breaker] circuit breaker not found") - - // ErrClockMustNotBeNil is returned when a nil clock is passed to NewCircuitBreakerManager - ErrClockMustNotBeNil = errors.New("[circuit breaker] clock must not be nil") - - // ErrConfigMustNotBeNil is returned when a nil config is passed to NewCircuitBreakerManager - ErrConfigMustNotBeNil = errors.New("[circuit breaker] config must not be nil") -) - -// CircuitBreakerConfig is the configuration that all the circuit breakers will use -type CircuitBreakerConfig struct { - // SampleRate is the time interval (in seconds) at which the data source - // is polled to determine the number successful and failed requests - SampleRate uint64 `json:"sample_rate"` - - // ErrorTimeout is the time (in seconds) after which a circuit breaker goes - // into the half-open state from the open state - ErrorTimeout uint64 `json:"error_timeout"` - - // FailureThreshold is the % of failed requests in the observability window - // after which the breaker will go into the open state - FailureThreshold float64 `json:"failure_threshold"` - - // FailureCount total number of failed requests in the observability window - FailureCount uint64 `json:"failure_count"` - - // SuccessThreshold is the % of successful requests in the observability window - // after which a circuit breaker in the half-open state will go into the closed state - SuccessThreshold uint64 `json:"success_threshold"` - - // ObservabilityWindow is how far back in time (in minutes) the data source is - // polled when determining the number successful and failed requests - ObservabilityWindow uint64 `json:"observability_window"` - - // NotificationThresholds These are the error counts after which we will send out notifications. - NotificationThresholds []uint64 `json:"notification_thresholds"` - - // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. - // E.g., after 10 consecutive transitions from half-open → open we should disable it. - ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold"` -} - -func (c *CircuitBreakerConfig) Validate() error { - var errs strings.Builder - - if c.SampleRate == 0 { - errs.WriteString("SampleRate must be greater than 0") - errs.WriteString("; ") - } - - if c.ErrorTimeout == 0 { - errs.WriteString("ErrorTimeout must be greater than 0") - errs.WriteString("; ") - } - - if c.FailureThreshold < 0 || c.FailureThreshold > 1 { - errs.WriteString("FailureThreshold must be between 0 and 1") - errs.WriteString("; ") - } - - if c.FailureCount == 0 { - errs.WriteString("FailureCount must be greater than 0") - errs.WriteString("; ") - } - - if c.SuccessThreshold == 0 { - errs.WriteString("SuccessThreshold must be greater than 0") - errs.WriteString("; ") - } - - if c.ObservabilityWindow == 0 { - errs.WriteString("ObservabilityWindow must be greater than 0") - errs.WriteString("; ") - } - - if c.ObservabilityWindow <= c.SampleRate { - errs.WriteString("ObservabilityWindow must be greater than the SampleRate") - errs.WriteString("; ") - } - - if len(c.NotificationThresholds) == 0 { - errs.WriteString("NotificationThresholds must contain at least one threshold") - errs.WriteString("; ") - } else { - for i := 0; i < len(c.NotificationThresholds); i++ { - if c.NotificationThresholds[i] == 0 { - errs.WriteString(fmt.Sprintf("Notification thresholds at index [%d] = %d must be greater than 0", i, c.NotificationThresholds[i])) - errs.WriteString("; ") - } - } - - for i := 0; i < len(c.NotificationThresholds)-1; i++ { - if c.NotificationThresholds[i] >= c.NotificationThresholds[i+1] { - errs.WriteString("NotificationThresholds should be in ascending order") - errs.WriteString("; ") - } - } - } - - if c.ConsecutiveFailureThreshold == 0 { - errs.WriteString("ConsecutiveFailureThreshold must be greater than 0") - errs.WriteString("; ") - } - - if errs.Len() > 0 { - return fmt.Errorf("config validation failed with errors: %s", errs.String()) - } - - return nil -} - -// State represents a state of a CircuitBreaker. -type State int - -// These are the states of a CircuitBreaker. -const ( - StateClosed State = iota - StateHalfOpen - StateOpen -) - -func (s State) String() string { - switch s { - case StateClosed: - return "closed" - case StateHalfOpen: - return "half-open" - case StateOpen: - return "open" - default: - return fmt.Sprintf("unknown state: %d", s) - } -} - // CircuitBreaker represents a circuit breaker type CircuitBreaker struct { Key string `json:"key"` @@ -193,334 +40,3 @@ func (b *CircuitBreaker) resetCircuitBreaker() { b.State = StateClosed b.ConsecutiveFailures = 0 } - -type PollResult struct { - Key string `json:"key" db:"key"` - Failures uint64 `json:"failures" db:"failures"` - Successes uint64 `json:"successes" db:"successes"` -} - -type CircuitBreakerManager struct { - config *CircuitBreakerConfig - clock clock.Clock - redis RedisClient -} - -type RedisClient interface { - Keys(ctx context.Context, pattern string) *redis.StringSliceCmd - Get(ctx context.Context, key string) *redis.StringCmd - MGet(ctx context.Context, keys ...string) *redis.SliceCmd - Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd - Scan(ctx context.Context, cursor uint64, match string, count int64) *redis.ScanCmd - TxPipeline() redis.Pipeliner -} - -func NewCircuitBreakerManager(client RedisClient) *CircuitBreakerManager { - defaultConfig := &CircuitBreakerConfig{ - SampleRate: 30, - ErrorTimeout: 30, - FailureThreshold: 0.1, - FailureCount: 10, - SuccessThreshold: 5, - ObservabilityWindow: 5, - NotificationThresholds: []uint64{5, 10}, - ConsecutiveFailureThreshold: 10, - } - - r := &CircuitBreakerManager{ - config: defaultConfig, - clock: clock.NewRealClock(), - redis: client, - } - - return r -} - -func (cb *CircuitBreakerManager) WithClock(c clock.Clock) (*CircuitBreakerManager, error) { - if cb.clock == nil { - return nil, ErrClockMustNotBeNil - } - - cb.clock = c - return cb, nil -} - -func (cb *CircuitBreakerManager) WithConfig(config *CircuitBreakerConfig) (*CircuitBreakerManager, error) { - if config == nil { - return nil, ErrConfigMustNotBeNil - } - - cb.config = config - - if err := config.Validate(); err != nil { - return nil, err - } - return cb, nil -} - -func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { - var keys []string - for i := range pollResults { - key := fmt.Sprintf("%s%s", prefix, pollResults[i].Key) - keys = append(keys, key) - pollResults[i].Key = key - } - - deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) - defer cancel() - - res, err := cb.redis.MGet(deadlineCtx, keys...).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - return nil - } - return err - } - - circuitBreakers := make([]CircuitBreaker, len(pollResults)) - for i := range res { - if res[i] == nil { - c := CircuitBreaker{ - State: StateClosed, - Key: pollResults[i].Key, - } - circuitBreakers[i] = c - continue - } - - c := CircuitBreaker{} - str, ok := res[i].(string) - if !ok { - log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) - - // the circuit breaker is corrupted, create a new one in its place - circuitBreakers[i] = CircuitBreaker{ - State: StateClosed, - Key: keys[i], - } - continue - } - - asBytes := []byte(str) - innerErr := msgpack.DecodeMsgPack(asBytes, &c) - if innerErr != nil { - return innerErr - } - - circuitBreakers[i] = c - } - - resultsMap := make(map[string]PollResult) - for _, result := range pollResults { - resultsMap[result.Key] = result - } - - circuitBreakerMap := make(map[string]CircuitBreaker, len(resultsMap)) - - for _, breaker := range circuitBreakers { - result := resultsMap[breaker.Key] - - breaker.TotalFailures = result.Failures - breaker.TotalSuccesses = result.Successes - breaker.Requests = breaker.TotalSuccesses + breaker.TotalFailures - - if breaker.Requests == 0 { - breaker.FailureRate = 0 - } else { - breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.Requests) - } - - if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { - breaker.resetCircuitBreaker() - } else if breaker.State == StateClosed && (breaker.FailureRate >= cb.config.FailureThreshold || breaker.TotalFailures >= cb.config.FailureCount) { - breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.ErrorTimeout) * time.Second)) - } - - if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { - breaker.toHalfOpen() - } - - circuitBreakerMap[breaker.Key] = breaker - } - - if err = cb.updateCircuitBreakers(ctx, circuitBreakerMap); err != nil { - log.WithError(err).Error("[circuit breaker] failed to update state") - return err - } - - return nil -} - -func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) (err error) { - deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) - defer cancel() - - pipe := cb.redis.TxPipeline() - for key, breaker := range breakers { - val, innerErr := breaker.String() - if innerErr != nil { - return innerErr - } - - if innerErr = pipe.Set(deadlineCtx, key, val, time.Duration(cb.config.ObservabilityWindow)*time.Minute).Err(); innerErr != nil { - return innerErr - } - } - - _, err = pipe.Exec(deadlineCtx) - if err != nil { - return err - } - - return nil -} - -func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { - deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) - defer cancel() - - keys, err := cb.redis.Keys(deadlineCtx, "breaker*").Result() - if err != nil { - return nil, err - } - - res, err := cb.redis.MGet(deadlineCtx, keys...).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - return nil, nil - } - return nil, err - } - - circuitBreakers := make([]CircuitBreaker, len(res)) - for i := range res { - c := CircuitBreaker{} - asBytes := []byte(res[i].(string)) - innerErr := msgpack.DecodeMsgPack(asBytes, &c) - if innerErr != nil { - return nil, innerErr - } - - circuitBreakers[i] = c - } - - return circuitBreakers, nil -} - -func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error { - switch b.State { - case StateOpen: - return ErrOpenState - case StateHalfOpen: - if b.TotalFailures > cb.config.FailureCount { - return ErrTooManyRequests - } - return nil - default: - return nil - } -} - -// CanExecute checks if the circuit breaker for a key will return an error for the current state. -// It will not return an error if it is in the closed state or half-open state when the failure -// threshold has not been reached, it will fail-open if the circuit breaker is not found -func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) error { - b, err := cb.getCircuitBreaker(ctx, key) - if err != nil { - return err - } - - if b != nil { - switch b.State { - case StateOpen, StateHalfOpen: - return cb.getCircuitBreakerError(*b) - default: - return nil - } - } - - return nil -} - -// getCircuitBreaker is used to get fetch the circuit breaker state, -// it fails open if the circuit breaker for that key is not found -func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) - defer cancel() - - bKey := fmt.Sprintf("%s%s", prefix, key) - res, err := cb.redis.Get(deadlineCtx, bKey).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - // a circuit breaker was not found for this key; - // it probably hasn't been created; - // we should fail open - return nil, nil - } - return nil, err - } - - err = msgpack.DecodeMsgPack([]byte(res), &c) - if err != nil { - return nil, err - } - - return c, nil -} - -// GetCircuitBreaker is used to get fetch the circuit breaker state, -// it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found -func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - deadlineCtx, cancel := context.WithDeadline(ctx, cb.clock.Now().Add(5*time.Second)) - defer cancel() - - bKey := fmt.Sprintf("%s%s", prefix, key) - res, err := cb.redis.Get(deadlineCtx, bKey).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - return nil, ErrCircuitBreakerNotFound - } - return nil, err - } - - err = msgpack.DecodeMsgPack([]byte(res), &c) - if err != nil { - return nil, err - } - - return c, nil -} - -func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { - // Get the failure and success counts from the last X minutes - pollResults, err := pollFunc(ctx, cb.config.ObservabilityWindow) - if err != nil { - return fmt.Errorf("poll function failed: %w", err) - } - - if len(pollResults) == 0 { - return nil // Nothing to update - } - - if err = cb.sampleStore(ctx, pollResults); err != nil { - return fmt.Errorf("[circuit breaker] failed to sample events and update state: %w", err) - } - - return nil -} - -func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { - ticker := time.NewTicker(time.Duration(cb.config.SampleRate) * time.Second) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { - log.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") - } - } - } -} diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go new file mode 100644 index 0000000000..28e7d91737 --- /dev/null +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -0,0 +1,378 @@ +package circuit_breaker + +import ( + "context" + "errors" + "fmt" + "github.com/frain-dev/convoy/pkg/clock" + "github.com/frain-dev/convoy/pkg/log" + "github.com/frain-dev/convoy/pkg/msgpack" + "time" +) + +const prefix = "breaker:" + +type PollFunc func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) +type CircuitBreakerOption func(cb *CircuitBreakerManager) error + +var ( + // ErrTooManyRequests is returned when the circuit breaker state is half open and the request count is over the failureThreshold + ErrTooManyRequests = errors.New("[circuit breaker] too many requests") + + // ErrOpenState is returned when the circuit breaker state is open + ErrOpenState = errors.New("[circuit breaker] circuit breaker is open") + + // ErrCircuitBreakerNotFound is returned when the circuit breaker is not found + ErrCircuitBreakerNotFound = errors.New("[circuit breaker] circuit breaker not found") + + // ErrClockMustNotBeNil is returned when a nil clock is passed to NewCircuitBreakerManager + ErrClockMustNotBeNil = errors.New("[circuit breaker] clock must not be nil") + + // ErrStoreMustNotBeNil is returned when a nil store is passed to NewCircuitBreakerManager + ErrStoreMustNotBeNil = errors.New("[circuit breaker] store must not be nil") + + // ErrConfigMustNotBeNil is returned when a nil config is passed to NewCircuitBreakerManager + ErrConfigMustNotBeNil = errors.New("[circuit breaker] config must not be nil") +) + +// State represents a state of a CircuitBreaker. +type State int + +// These are the states of a CircuitBreaker. +const ( + StateClosed State = iota + StateHalfOpen + StateOpen +) + +func (s State) String() string { + switch s { + case StateClosed: + return "closed" + case StateHalfOpen: + return "half-open" + case StateOpen: + return "open" + default: + return fmt.Sprintf("unknown state: %d", s) + } +} + +type PollResult struct { + Key string `json:"key" db:"key"` + Failures uint64 `json:"failures" db:"failures"` + Successes uint64 `json:"successes" db:"successes"` +} + +type CircuitBreakerManager struct { + config *CircuitBreakerConfig + clock clock.Clock + store CircuitBreakerStore +} + +func NewCircuitBreakerManager(options ...CircuitBreakerOption) (*CircuitBreakerManager, error) { + r := &CircuitBreakerManager{} + + for _, opt := range options { + err := opt(r) + if err != nil { + return r, err + } + } + + if r.store == nil { + return nil, ErrStoreMustNotBeNil + } + + if r.clock == nil { + return nil, ErrClockMustNotBeNil + } + + if r.config == nil { + return nil, ErrConfigMustNotBeNil + } + + return r, nil +} + +func StoreOption(store CircuitBreakerStore) CircuitBreakerOption { + return func(cb *CircuitBreakerManager) error { + if store == nil { + return ErrStoreMustNotBeNil + } + + cb.store = store + return nil + } +} + +func ClockOption(clock clock.Clock) CircuitBreakerOption { + return func(cb *CircuitBreakerManager) error { + if clock == nil { + return ErrClockMustNotBeNil + } + + cb.clock = clock + return nil + } +} + +func ConfigOption(config *CircuitBreakerConfig) CircuitBreakerOption { + return func(cb *CircuitBreakerManager) error { + if config == nil { + return ErrConfigMustNotBeNil + } + + if err := config.Validate(); err != nil { + return err + } + + cb.config = config + return nil + } +} + +func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + keys := make([]string, len(pollResults)) + for i := range pollResults { + key := fmt.Sprintf("%s%s", prefix, pollResults[i].Key) + keys[i] = key + pollResults[i].Key = key + } + + res, err := cb.store.GetMany(ctx, keys...) + if err != nil { + return err + } + + circuitBreakers := make([]CircuitBreaker, len(pollResults)) + for i := range res { + if res[i] == nil { + c := CircuitBreaker{ + State: StateClosed, + Key: pollResults[i].Key, + } + circuitBreakers[i] = c + continue + } + + c := CircuitBreaker{} + str, ok := res[i].(string) + if !ok { + log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) + + // the circuit breaker is corrupted, create a new one in its place + circuitBreakers[i] = CircuitBreaker{ + State: StateClosed, + Key: keys[i], + } + continue + } + + asBytes := []byte(str) + innerErr := msgpack.DecodeMsgPack(asBytes, &c) + if innerErr != nil { + return innerErr + } + + circuitBreakers[i] = c + } + + resultsMap := make(map[string]PollResult) + for _, result := range pollResults { + resultsMap[result.Key] = result + } + + circuitBreakerMap := make(map[string]CircuitBreaker, len(resultsMap)) + + for _, breaker := range circuitBreakers { + result := resultsMap[breaker.Key] + + breaker.TotalFailures = result.Failures + breaker.TotalSuccesses = result.Successes + breaker.Requests = breaker.TotalSuccesses + breaker.TotalFailures + + if breaker.Requests == 0 { + breaker.FailureRate = 0 + } else { + breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.Requests) + } + + if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { + breaker.resetCircuitBreaker() + } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && (breaker.FailureRate >= cb.config.FailureThreshold || breaker.TotalFailures >= cb.config.FailureCount) { + breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.ErrorTimeout) * time.Second)) + } + + if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { + breaker.toHalfOpen() + } + + circuitBreakerMap[breaker.Key] = breaker + } + + if err = cb.updateCircuitBreakers(ctx, circuitBreakerMap); err != nil { + log.WithError(err).Error("[circuit breaker] failed to update state") + return err + } + + return nil +} + +func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return cb.store.SetMany(ctx, breakers, time.Duration(cb.config.ObservabilityWindow)*time.Minute) +} + +func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + keys, err := cb.store.Keys(ctx, prefix) + if err != nil { + return nil, err + } + + res, err := cb.store.GetMany(ctx, keys...) + if err != nil { + return nil, err + } + + circuitBreakers := make([]CircuitBreaker, len(res)) + for i := range res { + c := CircuitBreaker{} + switch res[i].(type) { + case string: + asBytes := []byte(res[i].(string)) + innerErr := msgpack.DecodeMsgPack(asBytes, &c) + if innerErr != nil { + return nil, innerErr + } + case CircuitBreaker: + c = res[i].(CircuitBreaker) + } + + circuitBreakers[i] = c + } + + return circuitBreakers, nil +} + +func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error { + switch b.State { + case StateOpen: + return ErrOpenState + case StateHalfOpen: + if b.TotalFailures > cb.config.FailureCount { + return ErrTooManyRequests + } + return nil + default: + return nil + } +} + +// CanExecute checks if the circuit breaker for a key will return an error for the current state. +// It will not return an error if it is in the closed state or half-open state when the failure +// threshold has not been reached, it will fail-open if the circuit breaker is not found +func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) error { + b, err := cb.getCircuitBreaker(ctx, key) + if err != nil { + return err + } + + if b != nil { + switch b.State { + case StateOpen, StateHalfOpen: + return cb.getCircuitBreakerError(*b) + default: + return nil + } + } + + return nil +} + +// getCircuitBreaker is used to get fetch the circuit breaker state, +// it fails open if the circuit breaker for that key is not found +func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + bKey := fmt.Sprintf("%s%s", prefix, key) + res, err := cb.store.GetOne(ctx, bKey) + if err != nil { + if errors.Is(err, ErrCircuitBreakerNotFound) { + // a circuit breaker was not found for this key; + // it probably hasn't been created; + // we should fail open + return nil, nil + } + return nil, err + } + + err = msgpack.DecodeMsgPack([]byte(res), &c) + if err != nil { + return nil, err + } + + return c, nil +} + +// GetCircuitBreaker is used to get fetch the circuit breaker state, +// it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found +func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + bKey := fmt.Sprintf("%s%s", prefix, key) + res, err := cb.store.GetOne(ctx, bKey) + if err != nil { + return nil, err + } + + err = msgpack.DecodeMsgPack([]byte(res), &c) + if err != nil { + return nil, err + } + + return c, nil +} + +func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { + // Get the failure and success counts from the last X minutes + pollResults, err := pollFunc(ctx, cb.config.ObservabilityWindow) + if err != nil { + return fmt.Errorf("poll function failed: %w", err) + } + + if len(pollResults) == 0 { + return nil // Nothing to update + } + + if err = cb.sampleStore(ctx, pollResults); err != nil { + return fmt.Errorf("[circuit breaker] failed to sample events and update state: %w", err) + } + + return nil +} + +func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { + ticker := time.NewTicker(time.Duration(cb.config.SampleRate) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { + log.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") + } + } + } +} diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go new file mode 100644 index 0000000000..f59daf91ed --- /dev/null +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -0,0 +1,832 @@ +package circuit_breaker + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/frain-dev/convoy/pkg/clock" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func getRedis(t *testing.T) (client redis.UniversalClient, err error) { + t.Helper() + + opts, err := redis.ParseURL("redis://localhost:6379") + if err != nil { + return nil, err + } + + return redis.NewClient(opts), nil +} + +func pollResult(t *testing.T, key string, failureCount, successCount uint64) PollResult { + t.Helper() + + return PollResult{ + Key: key, + Failures: failureCount, + Successes: successCount, + } +} + +func TestCircuitBreakerManager(t *testing.T) { + ctx := context.Background() + + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + + re, err := getRedis(t) + require.NoError(t, err) + + store := NewRedisStore(re, testClock) + + keys, err := re.Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } + + c := &CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 3, + SuccessThreshold: 1, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10}, + ConsecutiveFailureThreshold: 10, + } + + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + require.NoError(t, err) + + endpointId := "endpoint-1" + pollResults := [][]PollResult{ + {pollResult(t, endpointId, 1, 0)}, + {pollResult(t, endpointId, 2, 0)}, + {pollResult(t, endpointId, 2, 1)}, + {pollResult(t, endpointId, 2, 2)}, + {pollResult(t, endpointId, 2, 3)}, + {pollResult(t, endpointId, 1, 4)}, + } + + for i := 0; i < len(pollResults); i++ { + innerErr := b.sampleStore(ctx, pollResults[i]) + require.NoError(t, innerErr) + + testClock.AdvanceTime(time.Minute) + } + + breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) + require.NoError(t, innerErr) + + require.Equal(t, breaker.State, StateClosed) +} + +func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { + ctx := context.Background() + + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + + re, err := getRedis(t) + require.NoError(t, err) + + store := NewRedisStore(re, testClock) + + keys, err := re.Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + err = re.Del(ctx, keys...).Err() + require.NoError(t, err) + + c := &CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 3, + SuccessThreshold: 1, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10}, + ConsecutiveFailureThreshold: 10, + } + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + require.NoError(t, err) + + endpoint1 := "endpoint-1" + endpoint2 := "endpoint-2" + pollResults := [][]PollResult{ + {pollResult(t, endpoint1, 1, 0)}, + {pollResult(t, endpoint1, 2, 0)}, + {pollResult(t, endpoint1, 2, 1), pollResult(t, endpoint2, 1, 0)}, + {pollResult(t, endpoint1, 2, 2), pollResult(t, endpoint2, 1, 1)}, + {pollResult(t, endpoint1, 2, 3), pollResult(t, endpoint2, 0, 2)}, + {pollResult(t, endpoint1, 1, 4), pollResult(t, endpoint2, 1, 1)}, + } + + for i := 0; i < len(pollResults); i++ { + err = b.sampleStore(ctx, pollResults[i]) + require.NoError(t, err) + + testClock.AdvanceTime(time.Minute) + } + + breakers, innerErr := b.loadCircuitBreakers(ctx) + require.NoError(t, innerErr) + + require.Len(t, breakers, 2) +} + +func TestCircuitBreakerManager_Transitions(t *testing.T) { + ctx := context.Background() + + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + + re, err := getRedis(t) + require.NoError(t, err) + + store := NewRedisStore(re, testClock) + + keys, err := store.Keys(ctx, "breaker*") + require.NoError(t, err) + + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } + + c := &CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 3, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10}, + ConsecutiveFailureThreshold: 10, + } + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + require.NoError(t, err) + + endpointId := "endpoint-1" + pollResults := [][]PollResult{ + {pollResult(t, endpointId, 1, 2)}, // Closed + {pollResult(t, endpointId, 3, 1)}, // Open (FailureCount reached) + {pollResult(t, endpointId, 0, 0)}, // Still Open + {pollResult(t, endpointId, 1, 1)}, // Half-Open (after ErrorTimeout) + {pollResult(t, endpointId, 0, 1)}, // Still Half-Open + {pollResult(t, endpointId, 0, 2)}, // Closed (SuccessThreshold reached) + {pollResult(t, endpointId, 4, 0)}, // Open (FailureThreshold reached) + } + + expectedStates := []State{ + StateClosed, + StateOpen, + StateOpen, + StateHalfOpen, + StateHalfOpen, + StateClosed, + StateOpen, + } + + for i, result := range pollResults { + err = b.sampleStore(ctx, result) + require.NoError(t, err) + + breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) + require.NoError(t, innerErr) + + require.Equal(t, expectedStates[i], breaker.State, "Iteration %d: expected state %v, got %v", i, expectedStates[i], breaker.State) + + if i == 2 { + // Advance time to trigger the transition to half-open + testClock.AdvanceTime(time.Duration(c.ErrorTimeout+1) * time.Second) + } else { + testClock.AdvanceTime(time.Second * 5) // Advance time by 5 seconds for other iterations + } + } +} + +func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { + ctx := context.Background() + + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + + re, err := getRedis(t) + require.NoError(t, err) + + store := NewRedisStore(re, testClock) + + keys, err := re.Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } + + c := &CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 3, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10}, + ConsecutiveFailureThreshold: 3, + } + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + require.NoError(t, err) + + endpointId := "endpoint-1" + pollResults := [][]PollResult{ + {pollResult(t, endpointId, 3, 1)}, // Open + {pollResult(t, endpointId, 1, 1)}, // Half-Open + {pollResult(t, endpointId, 3, 0)}, // Open + {pollResult(t, endpointId, 1, 1)}, // Half-Open + {pollResult(t, endpointId, 3, 0)}, // Open + } + + for _, result := range pollResults { + err = b.sampleStore(ctx, result) + require.NoError(t, err) + + testClock.AdvanceTime(time.Duration(c.ErrorTimeout+1) * time.Second) + } + + breaker, err := b.GetCircuitBreaker(ctx, endpointId) + require.NoError(t, err) + require.Equal(t, StateOpen, breaker.State) + require.Equal(t, uint64(3), breaker.ConsecutiveFailures) +} + +func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { + ctx := context.Background() + + testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + + re, err := getRedis(t) + require.NoError(t, err) + + store := NewRedisStore(re, testClock) + + keys, err := re.Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } + + c := &CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 3, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10}, + ConsecutiveFailureThreshold: 10, + } + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + require.NoError(t, err) + + endpoint1 := "endpoint-1" + endpoint2 := "endpoint-2" + endpoint3 := "endpoint-3" + + pollResults := [][]PollResult{ + {pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4)}, + {pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4)}, + {pollResult(t, endpoint1, 3, 1), pollResult(t, endpoint2, 1, 3), pollResult(t, endpoint3, 1, 5)}, + } + + for _, results := range pollResults { + err = b.sampleStore(ctx, results) + require.NoError(t, err) + + testClock.AdvanceTime(time.Duration(c.ErrorTimeout+1) * time.Second) + } + + breaker1, err := b.GetCircuitBreaker(ctx, endpoint1) + require.NoError(t, err) + require.Equal(t, StateOpen, breaker1.State) + + breaker2, err := b.GetCircuitBreaker(ctx, endpoint2) + require.NoError(t, err) + require.Equal(t, StateClosed, breaker2.State) + + breaker3, err := b.GetCircuitBreaker(ctx, endpoint3) + require.NoError(t, err) + require.Equal(t, StateClosed, breaker3.State) +} + +func TestCircuitBreakerManager_Config(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + t.Run("Success", func(t *testing.T) { + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + + require.NoError(t, err) + require.NotNil(t, manager) + require.Equal(t, mockStore, manager.store) + require.Equal(t, mockClock, manager.clock) + require.Equal(t, config, manager.config) + }) + + t.Run("Missing Store", func(t *testing.T) { + _, err := NewCircuitBreakerManager( + ClockOption(mockClock), + ConfigOption(config), + ) + + require.Error(t, err) + require.Equal(t, ErrStoreMustNotBeNil, err) + }) + + t.Run("Missing Clock", func(t *testing.T) { + _, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ConfigOption(config), + ) + + require.Error(t, err) + require.Equal(t, ErrClockMustNotBeNil, err) + }) + + t.Run("Missing Config", func(t *testing.T) { + _, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ) + + require.Error(t, err) + require.Equal(t, ErrConfigMustNotBeNil, err) + }) +} + +func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager := &CircuitBreakerManager{config: config} + + t.Run("Open State", func(t *testing.T) { + breaker := CircuitBreaker{State: StateOpen} + err := manager.getCircuitBreakerError(breaker) + require.Equal(t, ErrOpenState, err) + }) + + t.Run("Half-Open State with Too Many Failures", func(t *testing.T) { + breaker := CircuitBreaker{State: StateHalfOpen, TotalFailures: 6} + err := manager.getCircuitBreakerError(breaker) + require.Equal(t, ErrTooManyRequests, err) + }) + + t.Run("Half-Open State with Acceptable Failures", func(t *testing.T) { + breaker := CircuitBreaker{State: StateHalfOpen, TotalFailures: 4} + err := manager.getCircuitBreakerError(breaker) + require.NoError(t, err) + }) + + t.Run("Closed State", func(t *testing.T) { + breaker := CircuitBreaker{State: StateClosed} + err := manager.getCircuitBreakerError(breaker) + require.NoError(t, err) + }) +} + +func TestCircuitBreakerManager_SampleStore(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx := context.Background() + pollResults := []PollResult{ + {Key: "test1", Failures: 3, Successes: 7}, + {Key: "test2", Failures: 6, Successes: 4}, + } + + err = manager.sampleStore(ctx, pollResults) + require.NoError(t, err) + + // Check if circuit breakers were created and updated correctly + cb1, err := manager.GetCircuitBreaker(ctx, "test1") + require.NoError(t, err) + require.Equal(t, StateClosed, cb1.State) + require.Equal(t, uint64(10), cb1.Requests) + require.Equal(t, uint64(3), cb1.TotalFailures) + require.Equal(t, uint64(7), cb1.TotalSuccesses) + + cb2, err := manager.GetCircuitBreaker(ctx, "test2") + require.NoError(t, err) + require.Equal(t, StateOpen, cb2.State) + require.Equal(t, uint64(10), cb2.Requests) + require.Equal(t, uint64(6), cb2.TotalFailures) + require.Equal(t, uint64(4), cb2.TotalSuccesses) +} + +func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx := context.Background() + breakers := map[string]CircuitBreaker{ + "breaker:test1": { + Key: "breaker:test1", + State: StateClosed, + Requests: 10, + TotalFailures: 3, + TotalSuccesses: 7, + }, + "breaker:test2": { + Key: "breaker:test2", + State: StateOpen, + Requests: 10, + TotalFailures: 6, + TotalSuccesses: 4, + }, + } + + err = manager.updateCircuitBreakers(ctx, breakers) + require.NoError(t, err) + + // Check if circuit breakers were updated in the store + cb1, err := manager.GetCircuitBreaker(ctx, "test1") + require.NoError(t, err) + require.Equal(t, StateClosed, cb1.State) + require.Equal(t, uint64(10), cb1.Requests) + require.Equal(t, uint64(3), cb1.TotalFailures) + require.Equal(t, uint64(7), cb1.TotalSuccesses) + + cb2, err := manager.GetCircuitBreaker(ctx, "test2") + require.NoError(t, err) + require.Equal(t, StateOpen, cb2.State) + require.Equal(t, uint64(10), cb2.Requests) + require.Equal(t, uint64(6), cb2.TotalFailures) + require.Equal(t, uint64(4), cb2.TotalSuccesses) +} + +func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx := context.Background() + breakers := map[string]CircuitBreaker{ + "breaker:test1": { + Key: "test1", + State: StateClosed, + Requests: 10, + TotalFailures: 3, + TotalSuccesses: 7, + }, + "breaker:test2": { + Key: "test2", + State: StateOpen, + Requests: 10, + TotalFailures: 6, + TotalSuccesses: 4, + }, + } + + err = manager.updateCircuitBreakers(ctx, breakers) + require.NoError(t, err) + + loadedBreakers, err := manager.loadCircuitBreakers(ctx) + require.NoError(t, err) + require.Len(t, loadedBreakers, 2) + + // Check if loaded circuit breakers match the original ones + for _, cb := range loadedBreakers { + originalCB, exists := breakers["breaker:"+cb.Key] + require.True(t, exists) + require.Equal(t, originalCB.State, cb.State) + require.Equal(t, originalCB.Requests, cb.Requests) + require.Equal(t, originalCB.TotalFailures, cb.TotalFailures) + require.Equal(t, originalCB.TotalSuccesses, cb.TotalSuccesses) + } +} + +func TestCircuitBreakerManager_CanExecute(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx := context.Background() + + t.Run("Circuit Breaker Not Found", func(t *testing.T) { + err := manager.CanExecute(ctx, "non_existent") + require.NoError(t, err) + }) + + t.Run("Closed State", func(t *testing.T) { + cb := CircuitBreaker{ + Key: "test_closed", + State: StateClosed, + } + err := manager.store.SetOne(ctx, "breaker:test_closed", cb, time.Minute) + require.NoError(t, err) + + err = manager.CanExecute(ctx, "test_closed") + require.NoError(t, err) + }) + + t.Run("Open State", func(t *testing.T) { + cb := CircuitBreaker{ + Key: "test_open", + State: StateOpen, + } + err := manager.store.SetOne(ctx, "breaker:test_open", cb, time.Minute) + require.NoError(t, err) + + err = manager.CanExecute(ctx, "test_open") + require.Equal(t, ErrOpenState, err) + }) + + t.Run("Half-Open State with Too Many Failures", func(t *testing.T) { + cb := CircuitBreaker{ + Key: "test_half_open", + State: StateHalfOpen, + TotalFailures: 6, + } + err := manager.store.SetOne(ctx, "breaker:test_half_open", cb, time.Minute) + require.NoError(t, err) + + err = manager.CanExecute(ctx, "test_half_open") + require.Equal(t, ErrTooManyRequests, err) + }) + + t.Run("Half-Open State with Acceptable Failures", func(t *testing.T) { + cb := CircuitBreaker{ + Key: "test_half_open_ok", + State: StateHalfOpen, + TotalFailures: 4, + } + err := manager.store.SetOne(ctx, "breaker:test_half_open_ok", cb, time.Minute) + require.NoError(t, err) + + err = manager.CanExecute(ctx, "test_half_open_ok") + require.NoError(t, err) + }) +} + +func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx := context.Background() + + t.Run("Circuit Breaker Not Found", func(t *testing.T) { + _, err := manager.GetCircuitBreaker(ctx, "non_existent") + require.Equal(t, ErrCircuitBreakerNotFound, err) + }) + + t.Run("Circuit Breaker Found", func(t *testing.T) { + cb := CircuitBreaker{ + Key: "test_cb", + State: StateClosed, + Requests: 10, + TotalFailures: 3, + TotalSuccesses: 7, + } + err := manager.store.SetOne(ctx, "breaker:test_cb", cb, time.Minute) + require.NoError(t, err) + + retrievedCB, err := manager.GetCircuitBreaker(ctx, "test_cb") + require.NoError(t, err) + require.Equal(t, cb.Key, retrievedCB.Key) + require.Equal(t, cb.State, retrievedCB.State) + require.Equal(t, cb.Requests, retrievedCB.Requests) + require.Equal(t, cb.TotalFailures, retrievedCB.TotalFailures) + require.Equal(t, cb.TotalSuccesses, retrievedCB.TotalSuccesses) + }) +} + +func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx := context.Background() + + t.Run("Sample and Update Success", func(t *testing.T) { + pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { + return []PollResult{ + {Key: "test1", Failures: 3, Successes: 7}, + {Key: "test2", Failures: 6, Successes: 4}, + }, nil + } + + err := manager.sampleAndUpdate(ctx, pollFunc) + require.NoError(t, err) + + // Check if circuit breakers were created and updated correctly + cb1, err := manager.GetCircuitBreaker(ctx, "test1") + require.NoError(t, err) + require.Equal(t, StateClosed, cb1.State) + require.Equal(t, uint64(10), cb1.Requests) + require.Equal(t, uint64(3), cb1.TotalFailures) + require.Equal(t, uint64(7), cb1.TotalSuccesses) + + cb2, err := manager.GetCircuitBreaker(ctx, "test2") + require.NoError(t, err) + require.Equal(t, StateOpen, cb2.State) + require.Equal(t, uint64(10), cb2.Requests) + require.Equal(t, uint64(6), cb2.TotalFailures) + require.Equal(t, uint64(4), cb2.TotalSuccesses) + }) + + t.Run("Sample and Update with Empty Results", + func(t *testing.T) { + pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { + return []PollResult{}, nil + } + + err := manager.sampleAndUpdate(ctx, pollFunc) + require.NoError(t, err) + }) + + t.Run("Sample and Update with Poll Function Error", func(t *testing.T) { + pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { + return nil, errors.New("poll function error") + } + + err := manager.sampleAndUpdate(ctx, pollFunc) + require.Error(t, err) + require.Contains(t, err.Error(), "poll function failed") + }) +} + +func TestCircuitBreakerManager_Start(t *testing.T) { + mockStore := NewTestStore() + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + config := &CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(mockStore), + ClockOption(mockClock), + ConfigOption(config), + ) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + pollCount := 0 + pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { + pollCount++ + return []PollResult{ + {Key: "test", Failures: uint64(pollCount), Successes: 10 - uint64(pollCount)}, + }, nil + } + + go manager.Start(ctx, pollFunc) + + // Wait for a few poll cycles + time.Sleep(2500 * time.Millisecond) + + // Check if the circuit breaker was updated + cb, err := manager.GetCircuitBreaker(ctx, "test") + require.NoError(t, err) + require.NotNil(t, cb) + require.Equal(t, uint64(10), cb.Requests) + require.True(t, cb.TotalFailures > 0) + require.True(t, cb.TotalSuccesses > 0) + + // Ensure the poll function was called multiple times + require.True(t, pollCount > 1) +} diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index b3553bd80f..a93d59f11b 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -1,175 +1,79 @@ package circuit_breaker import ( - "context" - "github.com/frain-dev/convoy/pkg/clock" - "github.com/redis/go-redis/v9" + "github.com/frain-dev/convoy/pkg/msgpack" "github.com/stretchr/testify/require" "testing" "time" ) -func getRedis(t *testing.T) (client redis.UniversalClient, err error) { - t.Helper() - - opts, err := redis.ParseURL("redis://localhost:6379") - if err != nil { - return nil, err +func TestCircuitBreaker_String(t *testing.T) { + cb := &CircuitBreaker{ + Key: "test", + State: StateClosed, + Requests: 100, + FailureRate: 0.1, + WillResetAt: time.Now(), + TotalFailures: 10, + TotalSuccesses: 90, + ConsecutiveFailures: 2, } - return redis.NewClient(opts), nil -} - -func pollResult(t *testing.T, key string, failureCount, successCount uint64) PollResult { - t.Helper() - - return PollResult{ - Key: key, - Failures: failureCount, - Successes: successCount, - } -} - -func TestNewCircuitBreaker(t *testing.T) { - ctx := context.Background() + t.Run("Success", func(t *testing.T) { + result, err := cb.String() - re, err := getRedis(t) - require.NoError(t, err) - - keys, err := re.Keys(ctx, "breaker*").Result() - require.NoError(t, err) - - for i := range keys { - err = re.Del(ctx, keys[i]).Err() require.NoError(t, err) - } - - testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) - - c := &CircuitBreakerConfig{ - SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.1, - FailureCount: 3, - SuccessThreshold: 1, - ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, - ConsecutiveFailureThreshold: 10, - } - - b, err := NewCircuitBreakerManager(re).WithClock(testClock) - require.NoError(t, err) + require.NotEmpty(t, result) - b, err = b.WithConfig(c) - require.NoError(t, err) - - endpointId := "endpoint-1" - pollResults := [][]PollResult{ - { - pollResult(t, endpointId, 1, 0), - }, - { - pollResult(t, endpointId, 2, 0), - }, - { - pollResult(t, endpointId, 2, 1), - }, - { - pollResult(t, endpointId, 2, 2), - }, - { - pollResult(t, endpointId, 2, 3), - }, - { - pollResult(t, endpointId, 1, 4), - }, - } + // Decode the result back to a CircuitBreaker + var decodedCB CircuitBreaker + err = msgpack.DecodeMsgPack([]byte(result), &decodedCB) + require.NoError(t, err) - for i := 0; i < len(pollResults); i++ { - innerErr := b.sampleStore(ctx, pollResults[i]) - require.NoError(t, innerErr) + // Compare the decoded CircuitBreaker with the original + require.Equal(t, cb.Key, decodedCB.Key) + require.Equal(t, cb.State, decodedCB.State) + require.Equal(t, cb.Requests, decodedCB.Requests) + require.Equal(t, cb.FailureRate, decodedCB.FailureRate) + require.Equal(t, cb.WillResetAt.Unix(), decodedCB.WillResetAt.Unix()) + require.Equal(t, cb.TotalFailures, decodedCB.TotalFailures) + require.Equal(t, cb.TotalSuccesses, decodedCB.TotalSuccesses) + require.Equal(t, cb.ConsecutiveFailures, decodedCB.ConsecutiveFailures) + }) +} - testClock.AdvanceTime(time.Minute) +func TestCircuitBreaker_tripCircuitBreaker(t *testing.T) { + cb := &CircuitBreaker{ + State: StateClosed, + ConsecutiveFailures: 0, } - breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) - require.NoError(t, innerErr) + resetTime := time.Now().Add(30 * time.Second) + cb.tripCircuitBreaker(resetTime) - require.Equal(t, breaker.State, StateClosed) + require.Equal(t, StateOpen, cb.State) + require.Equal(t, resetTime, cb.WillResetAt) + require.Equal(t, uint64(1), cb.ConsecutiveFailures) } -func TestNewCircuitBreaker_AddNewBreakerMidway(t *testing.T) { - ctx := context.Background() - - re, err := getRedis(t) - require.NoError(t, err) - - keys, err := re.Keys(ctx, "breaker*").Result() - require.NoError(t, err) - - err = re.Del(ctx, keys...).Err() - require.NoError(t, err) - - testClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) - - c := &CircuitBreakerConfig{ - SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.1, - FailureCount: 3, - SuccessThreshold: 1, - ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, - ConsecutiveFailureThreshold: 10, +func TestCircuitBreaker_toHalfOpen(t *testing.T) { + cb := &CircuitBreaker{ + State: StateOpen, } - b, err := NewCircuitBreakerManager(re).WithClock(testClock) - require.NoError(t, err) - b, err = b.WithConfig(c) - require.NoError(t, err) - - endpoint1 := "endpoint-1" - endpoint2 := "endpoint-2" - pollResults := [][]PollResult{ - { - pollResult(t, endpoint1, 1, 0), - }, - { - pollResult(t, endpoint1, 2, 0), - }, - { - pollResult(t, endpoint1, 2, 1), - pollResult(t, endpoint2, 1, 0), - }, - { - pollResult(t, endpoint1, 2, 2), - pollResult(t, endpoint2, 1, 1), - }, - { - pollResult(t, endpoint1, 2, 3), - pollResult(t, endpoint2, 0, 2), - }, - { - pollResult(t, endpoint1, 1, 4), - pollResult(t, endpoint2, 1, 1), - }, - } + cb.toHalfOpen() - for i := 0; i < len(pollResults); i++ { - err = b.sampleStore(ctx, pollResults[i]) - require.NoError(t, err) - - if i > 1 { - breaker, innerErr := b.GetCircuitBreaker(ctx, endpoint2) - require.NoError(t, innerErr) - t.Logf("%+v\n", breaker) - } + require.Equal(t, StateHalfOpen, cb.State) +} - testClock.AdvanceTime(time.Minute) +func TestCircuitBreaker_resetCircuitBreaker(t *testing.T) { + cb := &CircuitBreaker{ + State: StateOpen, + ConsecutiveFailures: 5, } - breakers, innerErr := b.loadCircuitBreakers(ctx) - require.NoError(t, innerErr) + cb.resetCircuitBreaker() - require.Len(t, breakers, 2) + require.Equal(t, StateClosed, cb.State) + require.Equal(t, uint64(0), cb.ConsecutiveFailures) } diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go new file mode 100644 index 0000000000..ff37b24874 --- /dev/null +++ b/pkg/circuit_breaker/config.go @@ -0,0 +1,109 @@ +package circuit_breaker + +import ( + "fmt" + "strings" +) + +// CircuitBreakerConfig is the configuration that all the circuit breakers will use +type CircuitBreakerConfig struct { + // SampleRate is the time interval (in seconds) at which the data source + // is polled to determine the number successful and failed requests + SampleRate uint64 `json:"sample_rate"` + + // ErrorTimeout is the time (in seconds) after which a circuit breaker goes + // into the half-open state from the open state + ErrorTimeout uint64 `json:"error_timeout"` + + // FailureThreshold is the % of failed requests in the observability window + // after which the breaker will go into the open state + FailureThreshold float64 `json:"failure_threshold"` + + // FailureCount total number of failed requests in the observability window + FailureCount uint64 `json:"failure_count"` + + // SuccessThreshold is the % of successful requests in the observability window + // after which a circuit breaker in the half-open state will go into the closed state + SuccessThreshold uint64 `json:"success_threshold"` + + // ObservabilityWindow is how far back in time (in minutes) the data source is + // polled when determining the number successful and failed requests + ObservabilityWindow uint64 `json:"observability_window"` + + // NotificationThresholds These are the error counts after which we will send out notifications. + NotificationThresholds []uint64 `json:"notification_thresholds"` + + // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. + // E.g., after 10 consecutive transitions from half-open → open we should disable it. + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold"` +} + +func (c *CircuitBreakerConfig) Validate() error { + var errs strings.Builder + + if c.SampleRate == 0 { + errs.WriteString("SampleRate must be greater than 0") + errs.WriteString("; ") + } + + if c.ErrorTimeout == 0 { + errs.WriteString("ErrorTimeout must be greater than 0") + errs.WriteString("; ") + } + + if c.FailureThreshold < 0 || c.FailureThreshold > 1 { + errs.WriteString("FailureThreshold must be between 0 and 1") + errs.WriteString("; ") + } + + if c.FailureCount == 0 { + errs.WriteString("FailureCount must be greater than 0") + errs.WriteString("; ") + } + + if c.SuccessThreshold == 0 { + errs.WriteString("SuccessThreshold must be greater than 0") + errs.WriteString("; ") + } + + if c.ObservabilityWindow == 0 { + errs.WriteString("ObservabilityWindow must be greater than 0") + errs.WriteString("; ") + } + + // ObservabilityWindow is in minutes and SampleRate is in seconds + if (c.ObservabilityWindow * 60) <= c.SampleRate { + errs.WriteString("ObservabilityWindow must be greater than the SampleRate") + errs.WriteString("; ") + } + + if len(c.NotificationThresholds) == 0 { + errs.WriteString("NotificationThresholds must contain at least one threshold") + errs.WriteString("; ") + } else { + for i := 0; i < len(c.NotificationThresholds); i++ { + if c.NotificationThresholds[i] == 0 { + errs.WriteString(fmt.Sprintf("Notification thresholds at index [%d] = %d must be greater than 0", i, c.NotificationThresholds[i])) + errs.WriteString("; ") + } + } + + for i := 0; i < len(c.NotificationThresholds)-1; i++ { + if c.NotificationThresholds[i] >= c.NotificationThresholds[i+1] { + errs.WriteString("NotificationThresholds should be in ascending order") + errs.WriteString("; ") + } + } + } + + if c.ConsecutiveFailureThreshold == 0 { + errs.WriteString("ConsecutiveFailureThreshold must be greater than 0") + errs.WriteString("; ") + } + + if errs.Len() > 0 { + return fmt.Errorf("config validation failed with errors: %s", errs.String()) + } + + return nil +} diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go new file mode 100644 index 0000000000..6673c5b673 --- /dev/null +++ b/pkg/circuit_breaker/config_test.go @@ -0,0 +1,125 @@ +package circuit_breaker + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestCircuitBreakerConfig_Validate(t *testing.T) { + tests := []struct { + name string + config CircuitBreakerConfig + wantErr bool + }{ + { + name: "Valid Config", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + }, + wantErr: false, + }, + { + name: "Invalid SampleRate", + config: CircuitBreakerConfig{ + SampleRate: 0, + }, + wantErr: true, + }, + { + name: "Invalid ErrorTimeout", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 0, + }, + wantErr: true, + }, + { + name: "Invalid FailureThreshold", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 1.5, + }, + wantErr: true, + }, + { + name: "Invalid FailureCount", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 0, + }, + wantErr: true, + }, + { + name: "Invalid SuccessThreshold", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 0, + }, + wantErr: true, + }, + { + name: "Invalid ObservabilityWindow", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 0, + NotificationThresholds: []uint64{10, 20, 30}, + }, + wantErr: true, + }, + { + name: "Invalid NotificationThresholds", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{}, + }, + wantErr: true, + }, + { + name: "Invalid ConsecutiveFailureThreshold", + config: CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 0, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/circuit_breaker/metrics.go b/pkg/circuit_breaker/metrics.go new file mode 100644 index 0000000000..df93ceeb9a --- /dev/null +++ b/pkg/circuit_breaker/metrics.go @@ -0,0 +1,31 @@ +package circuit_breaker + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + circuitBreakerState = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "circuit_breaker_state", + Help: "The current state of the circuit breaker (0: Closed, 1: Half-Open, 2: Open)", + }, + []string{"key"}, + ) + + circuitBreakerRequests = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "circuit_breaker_requests_total", + Help: "The total number of requests processed by the circuit breaker", + }, + []string{"key", "result"}, + ) +) + +// Call updateMetrics in the sampleStore method after updating each circuit breaker +func (cb *CircuitBreakerManager) updateMetrics(breaker CircuitBreaker) { + circuitBreakerState.WithLabelValues(breaker.Key).Set(float64(breaker.State)) + circuitBreakerRequests.WithLabelValues(breaker.Key, "success").Add(float64(breaker.TotalSuccesses)) + circuitBreakerRequests.WithLabelValues(breaker.Key, "failure").Add(float64(breaker.TotalFailures)) +} diff --git a/pkg/circuit_breaker/store.go b/pkg/circuit_breaker/store.go new file mode 100644 index 0000000000..984303fb81 --- /dev/null +++ b/pkg/circuit_breaker/store.go @@ -0,0 +1,158 @@ +package circuit_breaker + +import ( + "context" + "errors" + "fmt" + "github.com/frain-dev/convoy/pkg/clock" + "github.com/redis/go-redis/v9" + "strings" + "sync" + "time" +) + +type CircuitBreakerStore interface { + Keys(context.Context, string) ([]string, error) + GetOne(context.Context, string) (string, error) + GetMany(context.Context, ...string) ([]interface{}, error) + SetOne(context.Context, string, interface{}, time.Duration) error + SetMany(context.Context, map[string]CircuitBreaker, time.Duration) error +} + +type RedisStore struct { + redis redis.UniversalClient + clock clock.Clock +} + +func NewRedisStore(redis redis.UniversalClient, clock clock.Clock) *RedisStore { + return &RedisStore{ + redis: redis, + clock: clock, + } +} + +// Keys returns all the keys used by the circuit breaker store +func (s *RedisStore) Keys(ctx context.Context, pattern string) ([]string, error) { + return s.redis.Keys(ctx, fmt.Sprintf("%s*", pattern)).Result() +} + +func (s *RedisStore) GetOne(ctx context.Context, key string) (string, error) { + key, err := s.redis.Get(ctx, key).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return "", ErrCircuitBreakerNotFound + } + return "", err + } + return key, nil +} + +func (s *RedisStore) GetMany(ctx context.Context, keys ...string) ([]any, error) { + res, err := s.redis.MGet(ctx, keys...).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return []any{}, nil + } + return nil, err + } + + return res, nil +} + +func (s *RedisStore) SetOne(ctx context.Context, key string, value interface{}, expiration time.Duration) error { + return s.redis.Set(ctx, key, value, expiration).Err() +} + +func (s *RedisStore) SetMany(ctx context.Context, breakers map[string]CircuitBreaker, ttl time.Duration) error { + pipe := s.redis.TxPipeline() + for key, breaker := range breakers { + val, innerErr := breaker.String() + if innerErr != nil { + return innerErr + } + + if innerErr = pipe.Set(ctx, key, val, ttl).Err(); innerErr != nil { + return innerErr + } + } + + _, err := pipe.Exec(ctx) + if err != nil { + return err + } + + return nil +} + +type TestStore struct { + store map[string]CircuitBreaker + mu *sync.RWMutex + clock clock.Clock +} + +func NewTestStore() *TestStore { + return &TestStore{ + store: make(map[string]CircuitBreaker), + mu: &sync.RWMutex{}, + clock: clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)), + } +} + +func (t TestStore) Keys(_ context.Context, s string) (keys []string, err error) { + t.mu.RLock() + defer t.mu.RUnlock() + + for key := range t.store { + if strings.HasPrefix(key, s) { + keys = append(keys, key) + } + } + + return keys, nil +} + +func (t TestStore) GetOne(_ context.Context, s string) (string, error) { + t.mu.RLock() + defer t.mu.RUnlock() + res, ok := t.store[s] + if !ok { + return "", ErrCircuitBreakerNotFound + } + + vv, err := res.String() + if err != nil { + return "", err + } + + return vv, nil +} + +func (t TestStore) GetMany(_ context.Context, keys ...string) (vals []interface{}, err error) { + t.mu.RLock() + defer t.mu.RUnlock() + for _, key := range keys { + if _, ok := t.store[key]; ok { + vals = append(vals, t.store[key]) + } else { + vals = append(vals, nil) + } + } + + return vals, nil +} + +func (t TestStore) SetOne(_ context.Context, key string, i interface{}, _ time.Duration) error { + t.mu.Lock() + defer t.mu.Unlock() + t.store[key] = i.(CircuitBreaker) + return nil +} + +func (t TestStore) SetMany(ctx context.Context, m map[string]CircuitBreaker, duration time.Duration) error { + for k, v := range m { + if err := t.SetOne(ctx, k, v, duration); err != nil { + return err + } + } + return nil +} diff --git a/pkg/circuit_breaker/store_test.go b/pkg/circuit_breaker/store_test.go new file mode 100644 index 0000000000..6b599b2bcd --- /dev/null +++ b/pkg/circuit_breaker/store_test.go @@ -0,0 +1,292 @@ +package circuit_breaker + +import ( + "context" + "testing" + "time" + + "github.com/frain-dev/convoy/pkg/clock" + "github.com/stretchr/testify/require" +) + +func TestRedisStore_Keys(t *testing.T) { + ctx := context.Background() + redisClient, err := getRedis(t) + require.NoError(t, err) + + mockClock := clock.NewSimulatedClock(time.Now()) + store := NewRedisStore(redisClient, mockClock) + + // Clean up any existing keys + existingKeys, err := redisClient.Keys(ctx, "test_keys*").Result() + require.NoError(t, err) + if len(existingKeys) > 0 { + err = redisClient.Del(ctx, existingKeys...).Err() + require.NoError(t, err) + } + + // Set up test data + testKeys := []string{"test_keys:1", "test_keys:2", "test_keys:3"} + for _, key := range testKeys { + err = redisClient.Set(ctx, key, "value", time.Minute).Err() + require.NoError(t, err) + } + + // Test Keys method + keys, err := store.Keys(ctx, "test_keys") + require.NoError(t, err) + require.ElementsMatch(t, testKeys, keys) + + // Clean up + err = redisClient.Del(ctx, testKeys...).Err() + require.NoError(t, err) +} + +func TestRedisStore_GetOne(t *testing.T) { + ctx := context.Background() + redisClient, err := getRedis(t) + require.NoError(t, err) + + mockClock := clock.NewSimulatedClock(time.Now()) + store := NewRedisStore(redisClient, mockClock) + + t.Run("Existing Key", func(t *testing.T) { + key := "test_get_one:existing" + value := "test_value" + err = redisClient.Set(ctx, key, value, time.Minute).Err() + require.NoError(t, err) + + result, err := store.GetOne(ctx, key) + require.NoError(t, err) + require.Equal(t, value, result) + + err = redisClient.Del(ctx, key).Err() + require.NoError(t, err) + }) + + t.Run("Non-existing Key", func(t *testing.T) { + key := "test_get_one:non_existing" + _, err := store.GetOne(ctx, key) + require.Equal(t, ErrCircuitBreakerNotFound, err) + }) +} + +func TestRedisStore_GetMany(t *testing.T) { + ctx := context.Background() + redisClient, err := getRedis(t) + require.NoError(t, err) + + mockClock := clock.NewSimulatedClock(time.Now()) + store := NewRedisStore(redisClient, mockClock) + + // Set up test data + testData := map[string]string{ + "test_get_many:1": "value1", + "test_get_many:2": "value2", + "test_get_many:3": "value3", + } + for key, value := range testData { + err = redisClient.Set(ctx, key, value, time.Minute).Err() + require.NoError(t, err) + } + + keys := []string{"test_get_many:1", "test_get_many:2", "test_get_many:3", "test_get_many:non_existing"} + results, err := store.GetMany(ctx, keys...) + require.NoError(t, err) + require.Len(t, results, 4) + + for i, key := range keys { + if i < 3 { + require.Equal(t, testData[key], results[i]) + } else { + require.Nil(t, results[i]) + } + } + + // Clean up + err = redisClient.Del(ctx, "test_get_many:1", "test_get_many:2", "test_get_many:3").Err() + require.NoError(t, err) +} + +func TestRedisStore_SetOne(t *testing.T) { + ctx := context.Background() + redisClient, err := getRedis(t) + require.NoError(t, err) + + mockClock := clock.NewSimulatedClock(time.Now()) + store := NewRedisStore(redisClient, mockClock) + + key := "test_set_one" + value := "test_value" + expiration := time.Minute + + err = store.SetOne(ctx, key, value, expiration) + require.NoError(t, err) + + // Verify the value was set + result, err := redisClient.Get(ctx, key).Result() + require.NoError(t, err) + require.Equal(t, value, result) + + // Verify the expiration was set + ttl, err := redisClient.TTL(ctx, key).Result() + require.NoError(t, err) + require.True(t, ttl > 0 && ttl <= expiration) + + // Clean up + err = redisClient.Del(ctx, key).Err() + require.NoError(t, err) +} + +func TestRedisStore_SetMany(t *testing.T) { + ctx := context.Background() + redisClient, err := getRedis(t) + require.NoError(t, err) + + mockClock := clock.NewSimulatedClock(time.Now()) + store := NewRedisStore(redisClient, mockClock) + + breakers := map[string]CircuitBreaker{ + "test_set_many:1": { + Key: "test_set_many:1", + State: StateClosed, + }, + "test_set_many:2": { + Key: "test_set_many:2", + State: StateOpen, + }, + } + expiration := time.Minute + + err = store.SetMany(ctx, breakers, expiration) + require.NoError(t, err) + + // Verify the values were set + for key, breaker := range breakers { + result, err := redisClient.Get(ctx, key).Result() + require.NoError(t, err) + + expectedValue, err := breaker.String() + require.NoError(t, err) + require.Equal(t, expectedValue, result) + + // Verify the expiration was set + ttl, err := redisClient.TTL(ctx, key).Result() + require.NoError(t, err) + require.True(t, ttl > 0 && ttl <= expiration) + } + + // Clean up + keys := make([]string, 0, len(breakers)) + for key := range breakers { + keys = append(keys, key) + } + err = redisClient.Del(ctx, keys...).Err() + require.NoError(t, err) +} + +func TestTestStore_Keys(t *testing.T) { + store := NewTestStore() + ctx := context.Background() + + // Add some test data + store.store["test:1"] = CircuitBreaker{Key: "test:1"} + store.store["test:2"] = CircuitBreaker{Key: "test:2"} + store.store["other:1"] = CircuitBreaker{Key: "other:1"} + + keys, err := store.Keys(ctx, "test") + require.NoError(t, err) + require.ElementsMatch(t, []string{"test:1", "test:2"}, keys) +} + +func TestTestStore_GetOne(t *testing.T) { + store := NewTestStore() + ctx := context.Background() + + t.Run("Existing Key", func(t *testing.T) { + cb := CircuitBreaker{Key: "test", State: StateClosed} + store.store["test"] = cb + + result, err := store.GetOne(ctx, "test") + require.NoError(t, err) + + expectedValue, _ := cb.String() + require.Equal(t, expectedValue, result) + }) + + t.Run("Non-existing Key", func(t *testing.T) { + _, err := store.GetOne(ctx, "non_existing") + require.Equal(t, ErrCircuitBreakerNotFound, err) + }) +} + +func TestTestStore_GetMany(t *testing.T) { + store := NewTestStore() + ctx := context.Background() + + cb1 := CircuitBreaker{Key: "test1", State: StateClosed} + cb2 := CircuitBreaker{Key: "test2", State: StateOpen} + store.store["test1"] = cb1 + store.store["test2"] = cb2 + + results, err := store.GetMany(ctx, "test1", "test2", "non_existing") + require.NoError(t, err) + require.Len(t, results, 3) + + require.Equal(t, cb1, results[0]) + require.Equal(t, cb2, results[1]) + require.Nil(t, results[2]) +} + +func TestTestStore_SetOne(t *testing.T) { + store := NewTestStore() + ctx := context.Background() + + cb := CircuitBreaker{Key: "test", State: StateClosed} + err := store.SetOne(ctx, "test", cb, time.Minute) + require.NoError(t, err) + + storedCB, ok := store.store["test"] + require.True(t, ok) + require.Equal(t, cb, storedCB) +} + +func TestTestStore_SetMany(t *testing.T) { + store := NewTestStore() + ctx := context.Background() + + breakers := map[string]CircuitBreaker{ + "test1": {Key: "test1", State: StateClosed}, + "test2": {Key: "test2", State: StateOpen}, + } + + err := store.SetMany(ctx, breakers, time.Minute) + require.NoError(t, err) + + for key, cb := range breakers { + storedCB, ok := store.store[key] + require.True(t, ok) + require.Equal(t, cb, storedCB) + } +} + +func TestTestStore_Concurrency(t *testing.T) { + store := NewTestStore() + ctx := context.Background() + + // Test concurrent reads and writes + go func() { + for i := 0; i < 100; i++ { + store.SetOne(ctx, "key", CircuitBreaker{Key: "key", State: StateClosed}, time.Minute) + } + }() + + go func() { + for i := 0; i < 100; i++ { + store.GetOne(ctx, "key") + } + }() + + // If there's a race condition, this test might panic or deadlock + time.Sleep(100 * time.Millisecond) +} From 7080d15387eb208633cb1957c5b5cc26fd6fbf2a Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 30 Aug 2024 12:00:31 +0200 Subject: [PATCH 15/48] chore: update config tests --- config/config_test.go | 30 +++++++++++++++++++++++++ database/postgres/configuration_test.go | 19 ++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 159a20eea2..f8c834b1fa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -125,6 +125,16 @@ func TestLoadConfig(t *testing.T) { Policy: "720h", IsRetentionPolicyEnabled: true, }, + CircuitBreaker: CircuitBreakerConfiguration{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{5, 10}, + ConsecutiveFailureThreshold: 10, + }, Server: ServerConfiguration{ HTTP: HTTPServerConfiguration{ Port: 80, @@ -196,6 +206,16 @@ func TestLoadConfig(t *testing.T) { Port: 5432, SetConnMaxLifetime: 3600, }, + CircuitBreaker: CircuitBreakerConfiguration{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{5, 10}, + ConsecutiveFailureThreshold: 10, + }, Redis: RedisConfiguration{ Scheme: "redis", Host: "localhost", @@ -262,6 +282,16 @@ func TestLoadConfig(t *testing.T) { Host: "localhost:5005", RetentionPolicy: RetentionPolicyConfiguration{Policy: "720h"}, ConsumerPoolSize: 100, + CircuitBreaker: CircuitBreakerConfiguration{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{5, 10}, + ConsecutiveFailureThreshold: 10, + }, Database: DatabaseConfiguration{ Type: PostgresDatabaseProvider, Scheme: "postgres", diff --git a/database/postgres/configuration_test.go b/database/postgres/configuration_test.go index cfe2ecf51c..e51189ef14 100644 --- a/database/postgres/configuration_test.go +++ b/database/postgres/configuration_test.go @@ -6,6 +6,7 @@ package postgres import ( "context" "errors" + "github.com/lib/pq" "testing" "time" @@ -93,10 +94,6 @@ func generateConfig() *datastore.Configuration { UID: ulid.Make().String(), IsAnalyticsEnabled: true, IsSignupEnabled: false, - RetentionPolicy: &datastore.RetentionPolicyConfiguration{ - Policy: "720h", - IsRetentionPolicyEnabled: true, - }, StoragePolicy: &datastore.StoragePolicyConfiguration{ Type: datastore.OnPrem, S3: &datastore.S3Storage{ @@ -112,5 +109,19 @@ func generateConfig() *datastore.Configuration { Path: null.NewString("path", true), }, }, + RetentionPolicy: &datastore.RetentionPolicyConfiguration{ + Policy: "720h", + IsRetentionPolicyEnabled: true, + }, + CircuitBreakerConfig: &datastore.CircuitBreakerConfig{ + SampleRate: 30, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 10, + SuccessThreshold: 5, + ObservabilityWindow: 5, + NotificationThresholds: pq.Int64Array{5, 10}, + ConsecutiveFailureThreshold: 10, + }, } } From 9c7c646fbf922b57d6b58d13f1a5fcf3be5be4f2 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 2 Sep 2024 09:50:54 +0200 Subject: [PATCH 16/48] chore: update tests --- worker/task/process_event_delivery_test.go | 24 ++++++++++++++++++- .../task/process_retry_event_delivery_test.go | 24 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 4153ec0455..89b4f4c3c4 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -4,8 +4,11 @@ import ( "context" "encoding/json" "github.com/frain-dev/convoy/net" + cb "github.com/frain-dev/convoy/pkg/circuit_breaker" + "github.com/frain-dev/convoy/pkg/clock" "github.com/stretchr/testify/require" "testing" + "time" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth/realm_chain" @@ -827,7 +830,26 @@ func TestProcessEventDelivery(t *testing.T) { dispatcher, err := net.NewDispatcher("", false) require.NoError(t, err) - processFn := ProcessEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo) + mockStore := cb.NewTestStore() + mockClock := clock.NewSimulatedClock(time.Now()) + breakerConfig := &cb.CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := cb.NewCircuitBreakerManager( + cb.StoreOption(mockStore), + cb.ClockOption(mockClock), + cb.ConfigOption(breakerConfig), + ) + + processFn := ProcessEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) payload := EventDelivery{ EventDeliveryID: tc.msg.UID, diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 55f2570fbb..f531fd43a1 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -5,8 +5,11 @@ import ( "encoding/json" "fmt" "github.com/frain-dev/convoy/net" + cb "github.com/frain-dev/convoy/pkg/circuit_breaker" + "github.com/frain-dev/convoy/pkg/clock" "github.com/stretchr/testify/require" "testing" + "time" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth/realm_chain" @@ -826,7 +829,26 @@ func TestProcessRetryEventDelivery(t *testing.T) { dispatcher, err := net.NewDispatcher("", false) require.NoError(t, err) - processFn := ProcessRetryEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo) + mockStore := cb.NewTestStore() + mockClock := clock.NewSimulatedClock(time.Now()) + breakerConfig := &cb.CircuitBreakerConfig{ + SampleRate: 1, + ErrorTimeout: 30, + FailureThreshold: 0.5, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: []uint64{10, 20, 30}, + ConsecutiveFailureThreshold: 3, + } + + manager, err := cb.NewCircuitBreakerManager( + cb.StoreOption(mockStore), + cb.ClockOption(mockClock), + cb.ConfigOption(breakerConfig), + ) + + processFn := ProcessRetryEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) payload := EventDelivery{ EventDeliveryID: tc.msg.UID, From 1efcafef37f06c19a4d27322907b5939a578692f Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 2 Sep 2024 11:05:02 +0200 Subject: [PATCH 17/48] chore: implement distributed lock using redlock --- .../circuit_breaker_manager.go | 14 +++++ pkg/circuit_breaker/store.go | 54 +++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 28e7d91737..821165d8d5 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -11,6 +11,7 @@ import ( ) const prefix = "breaker:" +const mutexKey = "convoy:circuit_breaker:mutex" type PollFunc func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) type CircuitBreakerOption func(cb *CircuitBreakerManager) error @@ -344,6 +345,19 @@ func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key stri } func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { + mu, err := cb.store.Lock(ctx, mutexKey) + if err != nil { + log.WithError(err).Error("[circuit breaker] failed to acquire lock") + return err + } + + defer func() { + innerErr := cb.store.Unlock(ctx, mu) + if innerErr != nil { + log.WithError(innerErr).Error("[circuit breaker] failed to unlock mutex") + } + }() + // Get the failure and success counts from the last X minutes pollResults, err := pollFunc(ctx, cb.config.ObservabilityWindow) if err != nil { diff --git a/pkg/circuit_breaker/store.go b/pkg/circuit_breaker/store.go index 984303fb81..fc468dcfec 100644 --- a/pkg/circuit_breaker/store.go +++ b/pkg/circuit_breaker/store.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "github.com/frain-dev/convoy/pkg/clock" + "github.com/go-redsync/redsync/v4" + "github.com/go-redsync/redsync/v4/redis/goredis/v9" "github.com/redis/go-redis/v9" "strings" "sync" @@ -12,6 +14,8 @@ import ( ) type CircuitBreakerStore interface { + Lock(ctx context.Context, lockKey string) (*redsync.Mutex, error) + Unlock(ctx context.Context, mutex *redsync.Mutex) error Keys(context.Context, string) ([]string, error) GetOne(context.Context, string) (string, error) GetMany(context.Context, ...string) ([]interface{}, error) @@ -31,6 +35,38 @@ func NewRedisStore(redis redis.UniversalClient, clock clock.Clock) *RedisStore { } } +func (s *RedisStore) Lock(ctx context.Context, mutexKey string) (*redsync.Mutex, error) { + pool := goredis.NewPool(s.redis) + rs := redsync.New(pool) + + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + mutex := rs.NewMutex(mutexKey, redsync.WithExpiry(time.Second), redsync.WithTries(1)) + err := mutex.LockContext(ctx) + if err != nil { + return nil, fmt.Errorf("failed to obtain lock: %v", err) + } + + return mutex, nil +} + +func (s *RedisStore) Unlock(ctx context.Context, mutex *redsync.Mutex) error { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + ok, err := mutex.UnlockContext(ctx) + if !ok { + return errors.New("failed to release lock") + } + + if err != nil { + return fmt.Errorf("failed to release lock: %v", err) + } + + return nil +} + // Keys returns all the keys used by the circuit breaker store func (s *RedisStore) Keys(ctx context.Context, pattern string) ([]string, error) { return s.redis.Keys(ctx, fmt.Sprintf("%s*", pattern)).Result() @@ -98,7 +134,15 @@ func NewTestStore() *TestStore { } } -func (t TestStore) Keys(_ context.Context, s string) (keys []string, err error) { +func (t *TestStore) Lock(_ context.Context, _ string) (*redsync.Mutex, error) { + return nil, nil +} + +func (t *TestStore) Unlock(_ context.Context, _ *redsync.Mutex) error { + return nil +} + +func (t *TestStore) Keys(_ context.Context, s string) (keys []string, err error) { t.mu.RLock() defer t.mu.RUnlock() @@ -111,7 +155,7 @@ func (t TestStore) Keys(_ context.Context, s string) (keys []string, err error) return keys, nil } -func (t TestStore) GetOne(_ context.Context, s string) (string, error) { +func (t *TestStore) GetOne(_ context.Context, s string) (string, error) { t.mu.RLock() defer t.mu.RUnlock() res, ok := t.store[s] @@ -127,7 +171,7 @@ func (t TestStore) GetOne(_ context.Context, s string) (string, error) { return vv, nil } -func (t TestStore) GetMany(_ context.Context, keys ...string) (vals []interface{}, err error) { +func (t *TestStore) GetMany(_ context.Context, keys ...string) (vals []interface{}, err error) { t.mu.RLock() defer t.mu.RUnlock() for _, key := range keys { @@ -141,14 +185,14 @@ func (t TestStore) GetMany(_ context.Context, keys ...string) (vals []interface{ return vals, nil } -func (t TestStore) SetOne(_ context.Context, key string, i interface{}, _ time.Duration) error { +func (t *TestStore) SetOne(_ context.Context, key string, i interface{}, _ time.Duration) error { t.mu.Lock() defer t.mu.Unlock() t.store[key] = i.(CircuitBreaker) return nil } -func (t TestStore) SetMany(ctx context.Context, m map[string]CircuitBreaker, duration time.Duration) error { +func (t *TestStore) SetMany(ctx context.Context, m map[string]CircuitBreaker, duration time.Duration) error { for k, v := range m { if err := t.SetOne(ctx, k, v, duration); err != nil { return err From afd3878376f273b403dc711392312121e9438a92 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Thu, 5 Sep 2024 14:34:26 +0200 Subject: [PATCH 18/48] merge with main --- .github/workflows/build-image.yml | 128 +++- .github/workflows/do-deploy.yml | 34 - .github/workflows/go.yml | 26 +- .github/workflows/immune-test.yml | 83 --- .github/workflows/linter.yml | 2 +- .github/workflows/release-ee.yml | 78 -- .github/workflows/release.yml | 16 +- .publisher-ee.yml | 100 --- .publisher.yml | 70 -- CHANGELOG.md | 43 ++ LICENSE | 389 ++-------- Makefile | 3 + README.md | 2 +- VERSION | 2 +- api/api.go | 329 +++++---- api/handlers/endpoint.go | 2 + api/handlers/event.go | 5 + api/handlers/event_delivery.go | 22 + api/handlers/license.go | 21 + api/handlers/middleware.go | 30 + api/handlers/organisation.go | 1 + api/handlers/organisation_invite.go | 2 + api/handlers/project.go | 5 +- api/handlers/shim.go | 67 -- api/handlers/subscription.go | 12 + api/handlers/user.go | 6 +- api/ingest.go | 14 +- api/ingest_integration_test.go | 1 + api/models/project.go | 1 + api/server_suite_test.go | 21 +- api/types/types.go | 16 +- cmd/agent/agent.go | 20 +- cmd/bootstrap/bootstrap.go | 9 + cmd/ff/feature_flags.go | 33 + cmd/hooks/hooks.go | 80 ++- cmd/ingest/ingest.go | 3 +- cmd/main.go | 12 +- cmd/server/server.go | 15 +- cmd/worker/worker.go | 24 +- config/config.go | 31 +- configs/convoy.templ.json | 7 - configs/docker-compose.templ.yml | 26 +- configs/local/docker-compose.yml | 23 +- database/postgres/event_delivery.go | 4 + database/postgres/organisation.go | 19 +- database/postgres/organisation_member.go | 1 + database/postgres/organisation_test.go | 27 + database/postgres/project.go | 18 +- database/postgres/project_test.go | 26 + database/postgres/users.go | 120 +--- database/postgres/users_test.go | 95 +-- datastore/filter.go | 9 +- datastore/filter_test.go | 2 +- datastore/repository.go | 4 +- docker-compose.dev.yml | 15 - docs/docs.go | 5 +- docs/swagger.json | 3 + docs/swagger.yaml | 2 + docs/v3/openapi3.json | 3 + docs/v3/openapi3.yaml | 2 + ee/VERSION | 2 +- ee/cmd/main.go | 9 +- ee/cmd/server/server.go | 18 +- generate.go | 1 + go.mod | 68 +- go.sum | 142 ++-- internal/pkg/cli/cli.go | 15 +- internal/pkg/fflag/fflag.go | 87 ++- internal/pkg/fflag/fflag_test.go | 211 ++++++ internal/pkg/license/keygen/community.go | 55 ++ internal/pkg/license/keygen/community_test.go | 89 +++ internal/pkg/license/keygen/feature.go | 46 ++ internal/pkg/license/keygen/keygen.go | 472 +++++++++++++ internal/pkg/license/keygen/keygen_test.go | 668 ++++++++++++++++++ internal/pkg/license/license.go | 47 ++ internal/pkg/license/noop/noop.go | 100 +++ internal/pkg/metrics/data_plane.go | 23 +- internal/pkg/middleware/middleware.go | 16 +- internal/pkg/pubsub/amqp/client.go | 13 +- internal/pkg/pubsub/google/client.go | 9 +- internal/pkg/pubsub/ingest.go | 10 +- internal/pkg/pubsub/kafka/client.go | 11 +- internal/pkg/pubsub/pubsub.go | 22 +- internal/pkg/pubsub/sqs/client.go | 16 +- mocks/license.go | 349 +++++++++ mocks/repository.go | 61 +- net/dispatcher.go | 18 +- net/dispatcher_test.go | 72 ++ release.Dockerfile | 2 +- scripts/integration-test.sh | 2 + scripts/ui.sh | 2 +- services/create_endpoint.go | 13 + services/create_endpoint_test.go | 68 +- services/create_organisation.go | 17 +- services/create_organisation_test.go | 25 + services/create_subscription.go | 17 +- services/create_subscription_test.go | 98 +++ services/invite_user.go | 16 +- services/invite_user_test.go | 22 + services/process_invite.go | 14 + services/process_invite_test.go | 54 ++ services/project_service.go | 32 +- services/project_service_test.go | 149 +++- services/register_user.go | 19 +- services/register_user_test.go | 42 ++ services/update_endpoint.go | 18 +- services/update_endpoint_test.go | 47 ++ services/update_subscription.go | 14 +- services/update_subscription_test.go | 1 + sql/1724236900.sql | 4 +- testcon/direct_event_test.go | 24 +- ...test.go => docker_e2e_integration_test.go} | 39 +- ... => docker_e2e_integration_test_helper.go} | 64 +- testcon/fanout_event_test.go | 48 +- .../{convoy-test.json => convoy-docker.json} | 0 testcon/testdata/convoy-host.json | 56 ++ testcon/testdata/docker-compose-test.yml | 10 +- .../src/app/components/tag/tag.component.ts | 3 +- .../src/app/portal/portal.component.html | 50 +- .../create-endpoint.component.html | 30 +- .../create-endpoint.component.ts | 8 +- .../create-project-component.component.html | 34 +- .../create-project-component.component.ts | 30 +- .../create-project-component.module.ts | 4 +- .../create-source.component.html | 26 +- .../create-source/create-source.component.ts | 3 +- .../create-source/create-source.module.ts | 4 +- .../create-subscription.component.html | 26 +- .../create-subscription.component.ts | 3 +- .../create-subscription.module.ts | 4 +- .../event-delivery-filter.component.html | 16 +- .../event-delivery-filter.component.ts | 3 +- .../portal-links/portal-links.component.html | 257 +++---- .../portal-links/portal-links.component.ts | 5 +- .../pages/project/project.component.html | 31 +- .../pages/project/project.component.ts | 3 +- .../subscriptions.component.html | 6 +- .../subscriptions/subscriptions.component.ts | 3 +- .../pages/projects/projects.component.html | 20 +- .../pages/projects/projects.component.ts | 3 +- .../organisation-settings.component.html | 2 +- .../organisation-settings.component.ts | 3 +- .../pages/settings/settings.component.html | 16 +- .../pages/settings/settings.component.ts | 13 +- .../src/app/private/private.component.html | 8 +- .../src/app/private/private.component.ts | 12 +- .../src/app/public/login/login.component.html | 2 +- .../src/app/public/login/login.component.ts | 4 +- .../src/app/public/signup/signup.component.ts | 9 +- .../licenses/licenses.service.spec.ts | 16 + .../app/services/licenses/licenses.service.ts | 51 ++ .../src/assets/img/svg/page-locked.svg | 9 + web/ui/dashboard/src/index.html | 6 + .../task/process_broadcast_event_creation.go | 10 +- .../process_broadcast_event_creation_test.go | 2 +- worker/task/process_dynamic_event_creation.go | 8 +- .../process_dynamic_event_creation_test.go | 2 +- worker/task/process_event_creation.go | 26 +- worker/task/process_event_creation_test.go | 159 ++++- worker/task/process_event_delivery.go | 32 +- worker/task/process_event_delivery_test.go | 145 +++- worker/task/process_meta_event.go | 14 +- worker/task/process_meta_event_test.go | 13 +- .../task/process_retry_event_delivery_test.go | 166 ++++- worker/task/search_tokenizer.go | 13 + worker/task/testdata/Config/basic-convoy.json | 7 - 166 files changed, 5104 insertions(+), 1952 deletions(-) delete mode 100644 .github/workflows/do-deploy.yml delete mode 100644 .github/workflows/immune-test.yml delete mode 100644 .github/workflows/release-ee.yml delete mode 100644 .publisher-ee.yml create mode 100644 api/handlers/license.go create mode 100644 api/handlers/middleware.go delete mode 100644 api/handlers/shim.go create mode 100644 cmd/ff/feature_flags.go create mode 100644 internal/pkg/fflag/fflag_test.go create mode 100644 internal/pkg/license/keygen/community.go create mode 100644 internal/pkg/license/keygen/community_test.go create mode 100644 internal/pkg/license/keygen/feature.go create mode 100644 internal/pkg/license/keygen/keygen.go create mode 100644 internal/pkg/license/keygen/keygen_test.go create mode 100644 internal/pkg/license/license.go create mode 100644 internal/pkg/license/noop/noop.go create mode 100644 mocks/license.go rename testcon/{integration_test.go => docker_e2e_integration_test.go} (57%) rename testcon/{integration_test_helper.go => docker_e2e_integration_test_helper.go} (87%) rename testcon/testdata/{convoy-test.json => convoy-docker.json} (100%) create mode 100644 testcon/testdata/convoy-host.json create mode 100644 web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts create mode 100644 web/ui/dashboard/src/app/services/licenses/licenses.service.ts create mode 100644 web/ui/dashboard/src/assets/img/svg/page-locked.svg diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index c007963748..c7f4a93c87 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,4 +1,4 @@ -name: Build docker image +name: Build and Push Docker Images on: workflow_dispatch: @@ -6,31 +6,117 @@ on: name: description: "Manual workflow name" required: true - + push: + tags: + - v* + +env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} + IMAGE_NAME: getconvoy/convoy + RELEASE_VERSION: ${{ github.ref_name }} jobs: - deploy: - runs-on: "ubuntu-latest" + build_ui: + name: Build UI + runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + - name: Build Artifact + run: "make ui_install type=ce" + + - name: Archive Build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-without-markdown + path: | + web/ui/dashboard/dist + !web/ui/dashboard/dist/**/*.md + + build-and-push-arch: + runs-on: ubuntu-latest + needs: [build_ui] + strategy: + matrix: + include: + - arch: amd64 + platform: linux/amd64 + dockerfile: release.Dockerfile + - arch: arm64 + platform: linux/arm64 + dockerfile: release.Dockerfile + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: dist-without-markdown + path: api/ui/build + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.21 - - name: Checkout code - uses: actions/checkout@v2 + - name: Get and verify dependencies + run: go mod tidy && go mod download && go mod verify - - name: Get the version - id: get_version - run: echo ::set-output name=tag::$(cat VERSION) + - name: Go vet + run: go vet ./... - - name: Authenticate - uses: actions-hub/docker/login@master - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.PAT }} - DOCKER_REGISTRY_URL: ghcr.io + - name: Build app to make sure there are zero issues + run: | + export CGO_ENABLED=0 + export GOOS=linux + export GOARCH=${{ matrix.arch }} + go build -o convoy ./cmd + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_TOKEN }} + + - name: Build and push arch specific images + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-${{ matrix.arch }} + build-args: | + ARCH=${{ matrix.arch }} + + build-and-push-default: + runs-on: ubuntu-latest + needs: [build-and-push-arch] + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_TOKEN }} - - name: Build latest image - run: docker build -t ghcr.io/${GITHUB_REPOSITORY}:${{ steps.get_version.outputs.tag }} . + - name: Create and push manifest for latest + run: | + docker buildx imagetools create -t ${{ env.IMAGE_NAME }}:latest \ + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-amd64 \ + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-arm64 - - name: Push - uses: actions-hub/docker@master - with: - args: push ghcr.io/${GITHUB_REPOSITORY}:${{ steps.get_version.outputs.tag }} + - name: Create and push manifest for version + run: | + docker buildx imagetools create -t ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }} \ + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-amd64 \ + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-arm64 \ No newline at end of file diff --git a/.github/workflows/do-deploy.yml b/.github/workflows/do-deploy.yml deleted file mode 100644 index e90f9443b1..0000000000 --- a/.github/workflows/do-deploy.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: DigitalOcean Deploy - -on: - push: - branches: - - main - workflow_dispatch: - inputs: - name: - description: "Manual workflow name" - required: true - -jobs: - deploy: - runs-on: "ubuntu-latest" - env: - REPO: registry.digitalocean.com/convoy-deployer - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build image - run: docker build -t $REPO/convoy:edge -f Dockerfile.dev . - - - name: Install doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} - - - name: Log in to DigitalOcean Container Registry with short-lived credentials - run: doctl registry login --expiry-seconds 60 - - - name: Push image to DigitalOcean Container Registry - run: docker push $REPO/convoy:edge diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ace152246c..6b9179619c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,6 @@ jobs: os: [ubuntu-latest, macos-latest] postgres-version: ["15"] redis-version: ["6.2.6"] - typesense-version: ["0.24.0"] runs-on: ubuntu-latest services: @@ -36,12 +35,6 @@ jobs: redis-version: ${{ matrix.redis-version }} redis-port: 6379 - - name: Start Typesense v${{ matrix.typesense-version }} - uses: jirevwe/typesense-github-action@v1.0.1 - with: - typesense-version: ${{ matrix.typesense-version }} - typesense-api-key: some-api-key - - name: Get the version id: get_version run: echo ::set-output name=tag::$(echo ${GITHUB_SHA:8}) @@ -97,6 +90,19 @@ jobs: TEST_REDIS_SCHEME: redis TEST_REDIS_HOST: localhost TEST_REDIS_PORT: 6379 - TEST_TYPESENSE_HOST: http://localhost:8108 - TEST_TYPESENSE_API_KEY: some-api-key - TEST_SEARCH_TYPE: typesense + + - name: Run integration tests (with test containers) + run: make docker_e2e_tests + env: + TEST_LICENSE_KEY: ${{ secrets.CONVOY_TEST_LICENSE_KEY }} + TEST_DB_SCHEME: postgres + TEST_DB_HOST: localhost + TEST_DB_USERNAME: postgres + TEST_DB_PASSWORD: postgres + TEST_DB_DATABASE: convoy + TEST_DB_OPTIONS: sslmode=disable&connect_timeout=30 + TEST_DB_PORT: 5432 + TEST_REDIS_SCHEME: redis + TEST_REDIS_HOST: localhost + TEST_REDIS_PORT: 6379 + diff --git a/.github/workflows/immune-test.yml b/.github/workflows/immune-test.yml deleted file mode 100644 index 09d342f79c..0000000000 --- a/.github/workflows/immune-test.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Build and run immune tests -on: - push: - branches: - - main - pull_request: - -jobs: - test: - if: ${{ !(contains(github.head_ref, 'ui/')) || !(contains(github.head_ref, 'cms/')) }} - strategy: - matrix: - go-version: [1.16.x, 1.17.x] - immune-test-file-names: [] - immune-version: ["0.2.1"] - mongodb-version: ["4.0", "4.2", "4.4"] - redis-version: ["6.2.6"] - - runs-on: ubuntu-latest - steps: - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.4.1 - with: - mongodb-version: ${{ matrix.mongodb-version }} - - - name: Start Redis v${{ matrix.redis-version }} - uses: supercharge/redis-github-action@1.4.0 - with: - redis-version: ${{ matrix.redis-version }} - redis-port: 6379 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - name: Check out code - uses: actions/checkout@v2 - - - name: Pull immune - run: | - wget --output-document=./immune.tar.gz \ - https://github.com/frain-dev/immune/releases/download/v${{ matrix.immune-version }}/immune_${{ matrix.immune-version }}_linux_amd64.tar.gz - tar -xvzf ./immune.tar.gz - mv ./immune $(go env GOPATH)/bin/immune - - - name: Setup custom host for endpoint - run: echo "127.0.0.1 www.endpoint.url" | sudo tee -a /etc/hosts - - - name: Pull certgen - uses: danvixent/certgen-action@v0.1.6 - with: - output-folder: $(go env GOPATH)/bin - os: ${{ runner.os }} - certgen-version: 0.2.0 - - - name: Start convoy & run immune tests - env: - PORT: 5005 - CONVOY_RETRY_LIMIT: "3" - CONVOY_INTERVAL_SECONDS: "10" - CONVOY_SIGNATURE_HEADER: "X-Convoy-CI" - CONVOY_STRATEGY_TYPE: "default" - CONVOY_SIGNATURE_HASH: "SHA256" - CONVOY_DB_TYPE: "mongodb" - CONVOY_SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - CONVOY_DB_DSN: "mongodb://localhost:27017/testdb" - CONVOY_REDIS_DSN: "redis://localhost:6379" - CONVOY_QUEUE_PROVIDER: "redis" - IMMUNE_EVENT_TARGET_URL: https://www.endpoint.url:9098 - IMMUNE_SSL: true - run: | - ref=$(certgen -domains="www.endpoint.url,endpoint.url") - echo "$ref" - go run ./cmd server & - IFS=', ' read -ra array <<< "$ref" - echo "${array[0]}" - echo "${array[1]}" - export IMMUNE_SSL_CERT_FILE="${array[0]}" - export IMMUNE_SSL_KEY_FILE="${array[1]}" - sleep 70 - cd ./immune-test-files - immune run --config ./${{ matrix.immune-test-file-names }} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 1db58a862c..c192c46b11 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -8,7 +8,7 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: diff --git a/.github/workflows/release-ee.yml b/.github/workflows/release-ee.yml deleted file mode 100644 index 6c1e394151..0000000000 --- a/.github/workflows/release-ee.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Release EE Binaries - -on: - workflow_dispatch: - inputs: - name: - description: "Manual workflow name" - required: true - push: - tags: - # Release binary for every tag. - - v* - -jobs: - build_ui: - name: Build UI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build Artifact - run: "make ui_install type=ee" - - name: Archive Build artifacts - uses: actions/upload-artifact@v2 - with: - name: dist-without-markdown - path: | - web/ui/dashboard/dist - !web/ui/dashboard/dist/**/*.md - - release-matrix: - name: Release & Publish Go Binary - needs: [build_ui] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Download Build Artifact - uses: actions/download-artifact@v2 - with: - name: dist-without-markdown - path: api/ui/build - fetch-depth: 0 - - - uses: docker/login-action@v1 - name: Authenticate with Docker - with: - registry: docker.cloudsmith.io - username: ${{ secrets.CLOUDSMITH_USERNAME }} - password: ${{ secrets.CLOUDSMITH_API_KEY }} - - - uses: actions/setup-go@v2 - name: Setup go - with: - go-version: '1.21' - - - uses: docker/setup-qemu-action@v3 - name: Set up QEMU - - - uses: actions/setup-python@v3 - name: Setup Python - with: - python-version: '3.9' - - - name: Install Cloudsmith CLI - run: | - echo $(pip --version) - pip install --upgrade cloudsmith-cli - echo $(cloudsmith --version) - - - uses: goreleaser/goreleaser-action@v2 - name: Release, Upload & Publish - with: - version: latest - args: -f .publisher-ee.yml release --clean - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - REPO_NAME: ${{ github.repository }} - CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ebb4b5977b..197b3378c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: needs: [build_ui] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Download Build Artifact uses: actions/download-artifact@v2 with: @@ -40,22 +40,10 @@ jobs: path: api/ui/build fetch-depth: 0 - - uses: docker/login-action@v1 - name: Authenticate with Docker - with: - registry: docker.cloudsmith.io - username: ${{ secrets.CLOUDSMITH_USERNAME }} - password: ${{ secrets.CLOUDSMITH_API_KEY }} - - - uses: actions/setup-go@v2 - name: Setup go - with: - go-version: '1.21' - - uses: docker/setup-qemu-action@v3 name: Set up QEMU - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 name: Setup Python with: python-version: '3.9' diff --git a/.publisher-ee.yml b/.publisher-ee.yml deleted file mode 100644 index f8e722a27b..0000000000 --- a/.publisher-ee.yml +++ /dev/null @@ -1,100 +0,0 @@ -project_name: convoy - -before: - hooks: - - go mod tidy -builds: - - env: - - CGO_ENABLED=0 - main: ./ee/cmd - id: cobin - goos: - - linux - - darwin - - windows - goarch: - - amd64 - - arm64 - -# https://goreleaser.com/customization/archive/ -archives: - - name_template: "{{ .ProjectName}}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" - id: cobin-archive - builds: - - cobin - -dockers: - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-amd64" - use: buildx - goos: linux - goarch: amd64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.source=https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-arm64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-arm64" - use: buildx - goos: linux - goarch: arm64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/arm64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.source=https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-slim" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-slim" - goos: linux - goarch: amd64 - dockerfile: slim.Dockerfile - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.source=https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - -docker_manifests: - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-arm64" - - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-arm64" - -checksum: - name_template: "{{ .ProjectName}}_checksums.txt" - -release: - # Will not auto-publish the release on GitHub - disable: true diff --git a/.publisher.yml b/.publisher.yml index 345f25d394..e496c2a668 100644 --- a/.publisher.yml +++ b/.publisher.yml @@ -87,76 +87,6 @@ brews: name: homebrew-tools url_template: https://dl.cloudsmith.io/public/convoy/convoy/raw/versions/{{.Version}}/{{ .ArtifactName }} -dockers: - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-amd64" - use: buildx - goos: linux - goarch: amd64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.source=https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-arm64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-arm64" - use: buildx - goos: linux - goarch: arm64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/arm64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.source=https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-slim" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-slim" - use: buildx - goos: linux - goarch: amd64 - dockerfile: slim.Dockerfile - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.source=https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - -docker_manifests: - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-arm64" - - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-arm64" - checksum: name_template: "{{ .ProjectName}}_checksums.txt" diff --git a/CHANGELOG.md b/CHANGELOG.md index efa1bb6a68..9e1c5992f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,46 @@ +# 24.8.1 + +### Features +- Added end to end latency metrics #2079 +- Add support for inbound webhooks that use form data #1998 +- Added e2e test suite using test containers #2083 +- Add license feature gating #2114 #2132 #2134 +- Change License to Elastic License v2.0 #2124 + +### Enhancements +- Move retention policy to instance config #2011 +- Update event data openapi types #2088 +- Refactor agent, worker and ingest entry points #2082 +- Refactored Exponential Backoff Implementation #2073 +- Remove instance configuration page #2085 +- Set default signature value to advanced from UI #2090 +- Add fanout for pubsub ingest #2099 +- Events ingested in incoming projects would now respond with a 413 for oversized payloads #2095 +- The agent component can now bootstrap a fresh instance #2111 +- Don't return an error when an owner id has no registered endpoints #2112 +- Split delivery attempts from event deliveries #2092 +- Add auth to metrics and queue monitoring routes #2115 +- Updated integration test suite #2100 +- Refactored feature flags implementation #2105 +- Push docker images to DockerHub #2122 +- Add owner id to event delivery response #2129 + + +### Bug Fixes +- Fixed a bug in positional array filter #2086 +- Fix count & batch retry queries #2089 +- Fixed a bug where api responses from v2024-04-01 to v2024-01-01 were not properly migrated #2087 +- Update UI Dependencies #2097 +- Fixed a migration bug where default column values were not set #2103 +- Fixed a bug where the wrong delay duration was used when scheduling an event delivery for retry #2110 +- Fixed a bug where other events were retried from a portal link because the endpoint filter wasn't applied #2116 +- + + +# 24.6.4 + +- fixed a bug where the pubsub ingester won't start when there aren't any projects + # 24.6.3 ### Bug Fixes diff --git a/LICENSE b/LICENSE index c33dcc7c92..5a155b34d8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,354 +1,109 @@ -Mozilla Public License, version 2.0 +CONVOY - Copyright (C) 2021-2024 Frain Technologies INC. All rights reserved. -1. Definitions +BEFORE DOWNLOADING OR USING CONVOY (THE “SOFTWARE”), YOU SHOULD CAREFULLY +READ THE FOLLOWING LICENSE AGREEMENT THAT APPLIES TO YOUR USE OF THE SOFTWARE. +DOWNLOADING OR USING CONVOY ESTABLISHES A BINDING AGREEMENT BETWEEN FRAIN +TECHNOLOGIES INC. ("LICENSOR") AND YOU (INCLUDING YOUR COMPANY, IF +APPLICABLE). YOUR ACCEPTANCE OF THIS LICENSE AGREEMENT IS REQUIRED AS A +CONDITION TO PROCEEDING WITH YOUR DOWNLOAD OR USE OF THE SOFTWARE. -1.1. “Contributor” - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. +Elastic License 2.0 -1.2. “Contributor Version” +### Acceptance - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. +By using the software, you agree to all of the terms and conditions below. -1.3. “Contribution” - means Covered Software of a particular Contributor. +### Copyright License -1.4. “Covered Software” +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. -1.5. “Incompatible With Secondary Licenses” - means +### Limitations - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. -1.6. “Executable Form” +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. - means any form of the work other than Source Code Form. -1.7. “Larger Work” +### Patents - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. -1.8. “License” - means this document. +### Notices -1.9. “Licensable” +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. -1.10. “Modifications” - means any of the following: +### No Other Rights - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or +These terms do not imply any licenses other than those expressly granted in +these terms. - b. any new file in Source Code Form that contains any Covered Software. -1.11. “Patent Claims” of a Contributor +### Termination - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. -1.12. “Secondary License” - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. +### No Liability -1.13. “Source Code Form” +As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim. - means the form of the work preferred for making modifications. -1.14. “You” (or “Your”) +### Definitions - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. +**you** refers to the individual or entity agreeing to these terms. -2. License Grants and Conditions +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. 'control' means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. -2.1. Grants +**your licenses** are all the licenses granted to you for the software under +these terms. - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and - - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. - -2.2. Effective Date - - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. - -2.3. Limitations on Grant Scope - - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: - - a. for any code that a Contributor has removed from Covered Software; or - - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. - - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). - -2.4. Subsequent Licenses - - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). - -2.5. Representation - - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. - -2.6. Fair Use - - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. - - -3. Responsibilities - -3.1. Distribution of Source Form - - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - - If You distribute Covered Software in Executable Form then: - - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and - - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). - -3.4. Notices - - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. - -6. Disclaimer of Warranty - - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation - - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. - - -10. Versions of the License - -10.1. New Versions - - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. - -10.2. Effect of New Versions - - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. - -10.3. Modified Versions - - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. +**use** means anything you do with the software requiring one of your licenses. +**trademark** means trademarks, service marks, and similar rights. \ No newline at end of file diff --git a/Makefile b/Makefile index 0dd5865a72..7e858bb66c 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ integration_tests: go run ./cmd migrate up go test -tags integration -p 1 ./... +docker_e2e_tests: + go test -tags docker_testcon -p 1 ./... + generate_migration_time: @date +"%Y%m%d%H%M%S" diff --git a/README.md b/README.md index bc985e97bd..a437e8b852 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,4 @@ Convoy provides several key features: Thank you for your interest in contributing! Please refer to [CONTRIBUTING.md](https://github.com/frain-dev/convoy/blob/main/CONTRIBUTING.md) for guidance. For contributions to the Convoy dashboard, please refer to the [web/ui](https://github.com/frain-dev/convoy/tree/main/web/ui) directory. ## License -[Mozilla Public License v2.0](https://github.com/frain-dev/convoy/blob/main/LICENSE) +[Elastic License v2.0](https://github.com/frain-dev/convoy/blob/main/LICENSE) diff --git a/VERSION b/VERSION index 54e58cde83..7b9d3f8d2c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v24.6.1 +v24.8.1 diff --git a/api/api.go b/api/api.go index 5fe9dcc87b..70c3606ae5 100644 --- a/api/api.go +++ b/api/api.go @@ -2,6 +2,11 @@ package api import ( "embed" + "io/fs" + "net/http" + "path" + "strings" + authz "github.com/Subomi/go-authz" "github.com/frain-dev/convoy/api/handlers" "github.com/frain-dev/convoy/api/policies" @@ -13,13 +18,8 @@ import ( redisqueue "github.com/frain-dev/convoy/queue/redis" "github.com/go-chi/chi/v5" chiMiddleware "github.com/go-chi/chi/v5/middleware" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/subomi/requestmigrations" - "io/fs" - "net/http" - "path" - "strings" ) //go:embed ui/build @@ -119,7 +119,6 @@ func (a *ApplicationHandler) buildRouter() *chi.Mux { } func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { - router := a.buildRouter() handler := &handlers.Handler{A: a.A, RM: a.rm} @@ -145,46 +144,53 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { projectRouter.Route("/{projectID}", func(projectSubRouter chi.Router) { projectSubRouter.Get("/", handler.GetProject) - projectSubRouter.Put("/", handler.UpdateProject) + projectSubRouter.With(handler.RequireEnabledProject()).Put("/", handler.UpdateProject) projectSubRouter.Delete("/", handler.DeleteProject) projectSubRouter.Route("/endpoints", func(endpointSubRouter chi.Router) { - endpointSubRouter.Post("/", handler.CreateEndpoint) + endpointSubRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateEndpoint) endpointSubRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) endpointSubRouter.Route("/{endpointID}", func(e chi.Router) { e.Get("/", handler.GetEndpoint) - e.Put("/", handler.UpdateEndpoint) - e.Delete("/", handler.DeleteEndpoint) - e.Put("/expire_secret", handler.ExpireSecret) - e.Put("/pause", handler.PauseEndpoint) + + e.With(handler.RequireEnabledProject()).Use(handler.RequireEnabledProject()) + + e.With(handler.RequireEnabledProject()).Put("/", handler.UpdateEndpoint) + e.With(handler.RequireEnabledProject()).Delete("/", handler.DeleteEndpoint) + e.With(handler.RequireEnabledProject()).Put("/expire_secret", handler.ExpireSecret) + e.With(handler.RequireEnabledProject()).Put("/pause", handler.PauseEndpoint) }) }) // TODO(subomi): left this here temporarily till the data plane is stable. projectSubRouter.Route("/events", func(eventRouter chi.Router) { - // TODO(all): should the InstrumentPath change? - eventRouter.With(middleware.InstrumentPath("/events")).Post("/", handler.CreateEndpointEvent) - eventRouter.Post("/fanout", handler.CreateEndpointFanoutEvent) - eventRouter.Post("/broadcast", handler.CreateBroadcastEvent) - eventRouter.Post("/dynamic", handler.CreateDynamicEvent) - eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Route("/", func(writeEventRouter chi.Router) { + eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) - eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { - eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + // TODO(all): should the InstrumentPath change? + eventRouter.With(handler.RequireEnabledProject(), middleware.InstrumentPath("/events")).Post("/", handler.CreateEndpointEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/fanout", handler.CreateEndpointFanoutEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/broadcast", handler.CreateBroadcastEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/dynamic", handler.CreateDynamicEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/batchreplay", handler.BatchReplayEvents) + + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.With(handler.RequireEnabledProject()).Put("/replay", handler.ReplayEndpointEvent) + eventSubRouter.Get("/", handler.GetEndpointEvent) + }) }) }) projectSubRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/batchretry", handler.BatchRetryEventDelivery) eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliverySubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendEventDelivery) eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { deliveryRouter.Get("/", handler.GetDeliveryAttempts) @@ -194,45 +200,45 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) projectSubRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { - subscriptionRouter.Post("/", handler.CreateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSubscription) subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) subscriptionRouter.Post("/test_function", handler.TestSubscriptionFunction) subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) - subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Delete("/{subscriptionID}", handler.DeleteSubscription) subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) - subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Put("/{subscriptionID}", handler.UpdateSubscription) subscriptionRouter.Put("/{subscriptionID}/toggle_status", handler.ToggleSubscriptionStatus) }) projectSubRouter.Route("/sources", func(sourceRouter chi.Router) { - sourceRouter.Post("/", handler.CreateSource) + sourceRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSource) sourceRouter.Get("/{sourceID}", handler.GetSource) sourceRouter.With(middleware.Pagination).Get("/", handler.LoadSourcesPaged) sourceRouter.Post("/test_function", handler.TestSourceFunction) - sourceRouter.Put("/{sourceID}", handler.UpdateSource) - sourceRouter.Delete("/{sourceID}", handler.DeleteSource) + sourceRouter.With(handler.RequireEnabledProject()).Put("/{sourceID}", handler.UpdateSource) + sourceRouter.With(handler.RequireEnabledProject()).Delete("/{sourceID}", handler.DeleteSource) }) - projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { - portalLinkRouter.Post("/", handler.CreatePortalLink) - portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) - portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) - portalLinkRouter.Put("/{portalLinkID}", handler.UpdatePortalLink) - portalLinkRouter.Put("/{portalLinkID}/revoke", handler.RevokePortalLink) - }) + if handler.A.Licenser.PortalLinks() { + projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { + portalLinkRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreatePortalLink) + portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) + portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) + portalLinkRouter.With(handler.RequireEnabledProject()).Put("/{portalLinkID}", handler.UpdatePortalLink) + portalLinkRouter.With(handler.RequireEnabledProject()).Put("/{portalLinkID}/revoke", handler.RevokePortalLink) + }) + } projectSubRouter.Route("/meta-events", func(metaEventRouter chi.Router) { metaEventRouter.With(middleware.Pagination).Get("/", handler.GetMetaEventsPaged) metaEventRouter.Route("/{metaEventID}", func(metaEventSubRouter chi.Router) { metaEventSubRouter.Get("/", handler.GetMetaEvent) - metaEventSubRouter.Put("/resend", handler.ResendMetaEvent) + metaEventSubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendMetaEvent) }) }) }) }) - - r.HandleFunc("/*", handler.RedirectToProjects) }) }) @@ -241,6 +247,8 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { uiRouter.Use(middleware.JsonResponse) uiRouter.Use(chiMiddleware.Maybe(middleware.RequireAuth(), shouldAuthRoute)) + uiRouter.Get("/license/features", handler.GetLicenseFeatures) + uiRouter.Post("/users/forgot-password", handler.ForgotPassword) uiRouter.Post("/users/reset-password", handler.ResetPassword) uiRouter.Post("/users/verify_email", handler.VerifyEmail) @@ -300,50 +308,57 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { projectRouter.Route("/{projectID}", func(projectSubRouter chi.Router) { projectSubRouter.Get("/", handler.GetProject) - projectSubRouter.Put("/", handler.UpdateProject) - projectSubRouter.Delete("/", handler.DeleteProject) + projectSubRouter.With(handler.RequireEnabledProject()).Put("/", handler.UpdateProject) + projectSubRouter.With(handler.RequireEnabledProject()).Delete("/", handler.DeleteProject) projectSubRouter.Get("/stats", handler.GetProjectStatistics) projectSubRouter.Route("/security/keys", func(projectKeySubRouter chi.Router) { - projectKeySubRouter.Put("/regenerate", handler.RegenerateProjectAPIKey) + projectKeySubRouter.With(handler.RequireEnabledProject()).Put("/regenerate", handler.RegenerateProjectAPIKey) }) projectSubRouter.Route("/endpoints", func(endpointSubRouter chi.Router) { - endpointSubRouter.Post("/", handler.CreateEndpoint) + endpointSubRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateEndpoint) endpointSubRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) endpointSubRouter.Route("/{endpointID}", func(e chi.Router) { e.Get("/", handler.GetEndpoint) - e.Put("/", handler.UpdateEndpoint) - e.Delete("/", handler.DeleteEndpoint) - e.Put("/expire_secret", handler.ExpireSecret) - e.Put("/pause", handler.PauseEndpoint) + + e.With(handler.RequireEnabledProject()).Use(handler.RequireEnabledProject()) + + e.With(handler.RequireEnabledProject()).Put("/", handler.UpdateEndpoint) + e.With(handler.RequireEnabledProject()).Delete("/", handler.DeleteEndpoint) + e.With(handler.RequireEnabledProject()).Put("/expire_secret", handler.ExpireSecret) + e.With(handler.RequireEnabledProject()).Put("/pause", handler.PauseEndpoint) }) }) // TODO(subomi): left this here temporarily till the data plane is stable. projectSubRouter.Route("/events", func(eventRouter chi.Router) { - eventRouter.Post("/", handler.CreateEndpointEvent) - eventRouter.Post("/fanout", handler.CreateEndpointFanoutEvent) eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) + // TODO(all): should the InstrumentPath change? + eventRouter.With(handler.RequireEnabledProject(), middleware.InstrumentPath("/events")).Post("/", handler.CreateEndpointEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/fanout", handler.CreateEndpointFanoutEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/broadcast", handler.CreateBroadcastEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/dynamic", handler.CreateDynamicEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.With(handler.RequireEnabledProject()).Put("/replay", handler.ReplayEndpointEvent) eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) }) }) projectSubRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/batchretry", handler.BatchRetryEventDelivery) eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliverySubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendEventDelivery) eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { deliveryRouter.Get("/", handler.GetDeliveryAttempts) @@ -353,22 +368,22 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) projectSubRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { - subscriptionRouter.Post("/", handler.CreateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSubscription) subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) subscriptionRouter.Post("/test_function", handler.TestSubscriptionFunction) subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) - subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Delete("/{subscriptionID}", handler.DeleteSubscription) subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) - subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Put("/{subscriptionID}", handler.UpdateSubscription) }) projectSubRouter.Route("/sources", func(sourceRouter chi.Router) { - sourceRouter.Post("/", handler.CreateSource) + sourceRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSource) sourceRouter.Get("/{sourceID}", handler.GetSource) sourceRouter.With(middleware.Pagination).Get("/", handler.LoadSourcesPaged) sourceRouter.Post("/test_function", handler.TestSourceFunction) - sourceRouter.Put("/{sourceID}", handler.UpdateSource) - sourceRouter.Delete("/{sourceID}", handler.DeleteSource) + sourceRouter.With(handler.RequireEnabledProject()).Put("/{sourceID}", handler.UpdateSource) + sourceRouter.With(handler.RequireEnabledProject()).Delete("/{sourceID}", handler.DeleteSource) }) projectSubRouter.Route("/meta-events", func(metaEventRouter chi.Router) { @@ -376,17 +391,19 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { metaEventRouter.Route("/{metaEventID}", func(metaEventSubRouter chi.Router) { metaEventSubRouter.Get("/", handler.GetMetaEvent) - metaEventSubRouter.Put("/resend", handler.ResendMetaEvent) + metaEventSubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendMetaEvent) }) }) - projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { - portalLinkRouter.Post("/", handler.CreatePortalLink) - portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) - portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) - portalLinkRouter.Put("/{portalLinkID}", handler.UpdatePortalLink) - portalLinkRouter.Put("/{portalLinkID}/revoke", handler.RevokePortalLink) - }) + if handler.A.Licenser.PortalLinks() { + projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { + portalLinkRouter.Post("/", handler.CreatePortalLink) + portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) + portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) + portalLinkRouter.Put("/{portalLinkID}", handler.UpdatePortalLink) + portalLinkRouter.Put("/{portalLinkID}/revoke", handler.RevokePortalLink) + }) + } projectSubRouter.Route("/dashboard", func(dashboardRouter chi.Router) { dashboardRouter.Get("/summary", handler.GetDashboardSummary) @@ -403,70 +420,81 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) // Portal Link API. - router.Route("/portal-api", func(portalLinkRouter chi.Router) { - portalLinkRouter.Use(middleware.JsonResponse) - portalLinkRouter.Use(middleware.SetupCORS) - portalLinkRouter.Use(middleware.RequireAuth()) - - portalLinkRouter.Get("/portal_link", handler.GetPortalLink) - - portalLinkRouter.Route("/endpoints", func(endpointRouter chi.Router) { - endpointRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) - endpointRouter.Get("/{endpointID}", handler.GetEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Post("/", handler.CreateEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}", handler.UpdateEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Delete("/{endpointID}", handler.DeleteEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/pause", handler.PauseEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/expire_secret", handler.ExpireSecret) - }) + if handler.A.Licenser.PortalLinks() { + router.Route("/portal-api", func(portalLinkRouter chi.Router) { + portalLinkRouter.Use(middleware.JsonResponse) + portalLinkRouter.Use(middleware.SetupCORS) + portalLinkRouter.Use(middleware.RequireAuth()) + + portalLinkRouter.Get("/portal_link", handler.GetPortalLink) + + portalLinkRouter.Route("/endpoints", func(endpointRouter chi.Router) { + endpointRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) + endpointRouter.Get("/{endpointID}", handler.GetEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Post("/", handler.CreateEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}", handler.UpdateEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Delete("/{endpointID}", handler.DeleteEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/pause", handler.PauseEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/expire_secret", handler.ExpireSecret) + }) - // TODO(subomi): left this here temporarily till the data plane is stable. - portalLinkRouter.Route("/events", func(eventRouter chi.Router) { - eventRouter.Post("/", handler.CreateEndpointEvent) - eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) - eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) + // TODO(subomi): left this here temporarily till the data plane is stable. + portalLinkRouter.Route("/events", func(eventRouter chi.Router) { + eventRouter.Post("/", handler.CreateEndpointEvent) + eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) - eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { - eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.Get("/", handler.GetEndpointEvent) + eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + }) }) - }) - portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { - eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) - eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) + portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { + eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) + eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) - eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { - eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { + eventDeliverySubRouter.Get("/", handler.GetEventDelivery) + eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) - eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { - deliveryRouter.Get("/", handler.GetDeliveryAttempts) - deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { + deliveryRouter.Get("/", handler.GetDeliveryAttempts) + deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + }) }) }) + + portalLinkRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { + subscriptionRouter.Post("/", handler.CreateSubscription) + subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) + subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) + subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) + subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) + subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + }) }) + } - portalLinkRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { - subscriptionRouter.Post("/", handler.CreateSubscription) - subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) - subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) - subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) - subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) - subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + if a.A.Licenser.AsynqMonitoring() { + router.Route("/queue", func(asynqRouter chi.Router) { + asynqRouter.Use(middleware.RequireAuth()) + asynqRouter.Handle("/monitoring/*", a.A.Queue.(*redisqueue.RedisQueue).Monitor()) }) + } - }) + if a.A.Licenser.CanExportPrometheusMetrics() { + router.Route("/metrics", func(metricsRouter chi.Router) { + metricsRouter.Use(middleware.RequireAuth()) + metricsRouter.Get("/", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()}).ServeHTTP) + }) + } - router.Handle("/queue/monitoring/*", a.A.Queue.(*redisqueue.RedisQueue).Monitor()) - router.Handle("/metrics", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{})) router.HandleFunc("/*", reactRootHandler) - metrics.RegisterQueueMetrics(a.A.Queue, a.A.DB) - prometheus.MustRegister(metrics.RequestDuration()) a.Router = router return router @@ -474,7 +502,11 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { func (a *ApplicationHandler) BuildDataPlaneRoutes() *chi.Mux { router := a.buildRouter() - router.Handle("/metrics", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()})) + + router.Route("/metrics", func(metricsRouter chi.Router) { + metricsRouter.Use(middleware.RequireAuth()) + metricsRouter.Get("/", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()}).ServeHTTP) + }) // Ingestion API. router.Route("/ingest", func(ingestRouter chi.Router) { @@ -582,40 +614,42 @@ func (a *ApplicationHandler) BuildDataPlaneRoutes() *chi.Mux { }) // Portal Link API. - router.Route("/portal-api", func(portalLinkRouter chi.Router) { - portalLinkRouter.Use(middleware.JsonResponse) - portalLinkRouter.Use(middleware.SetupCORS) - portalLinkRouter.Use(middleware.RequireAuth()) - - portalLinkRouter.Route("/events", func(eventRouter chi.Router) { - eventRouter.Post("/", handler.CreateEndpointEvent) - eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) - eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) - - eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { - eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + if handler.A.Licenser.PortalLinks() { + router.Route("/portal-api", func(portalLinkRouter chi.Router) { + portalLinkRouter.Use(middleware.JsonResponse) + portalLinkRouter.Use(middleware.SetupCORS) + portalLinkRouter.Use(middleware.RequireAuth()) + + portalLinkRouter.Route("/events", func(eventRouter chi.Router) { + eventRouter.Post("/", handler.CreateEndpointEvent) + eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) + + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.Get("/", handler.GetEndpointEvent) + eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + }) }) - }) - portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { - eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) - eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) + portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { + eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) + eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) - eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { - eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { + eventDeliverySubRouter.Get("/", handler.GetEventDelivery) + eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) - eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { - deliveryRouter.Get("/", handler.GetDeliveryAttempts) - deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { + deliveryRouter.Get("/", handler.GetDeliveryAttempts) + deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + }) }) }) }) - }) + } a.Router = router @@ -665,6 +699,7 @@ var guestRoutes = []string{ "/users/verify_email", "/organisations/process_invite", "/ui/configuration/is_signup_enabled", + "/ui/license/features", } func shouldAuthRoute(r *http.Request) bool { diff --git a/api/handlers/endpoint.go b/api/handlers/endpoint.go index c57895c58f..28103ff580 100644 --- a/api/handlers/endpoint.go +++ b/api/handlers/endpoint.go @@ -75,6 +75,7 @@ func (h *Handler) CreateEndpoint(w http.ResponseWriter, r *http.Request) { EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), ProjectRepo: postgres.NewProjectRepo(h.A.DB, h.A.Cache), PortalLinkRepo: postgres.NewPortalLinkRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, E: e, ProjectID: project.UID, } @@ -281,6 +282,7 @@ func (h *Handler) UpdateEndpoint(w http.ResponseWriter, r *http.Request) { Cache: h.A.Cache, EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), ProjectRepo: postgres.NewProjectRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, E: e, Endpoint: endpoint, Project: project, diff --git a/api/handlers/event.go b/api/handlers/event.go index 54da990508..6ab7645305 100644 --- a/api/handlers/event.go +++ b/api/handlers/event.go @@ -477,6 +477,11 @@ func (h *Handler) GetEventsPaged(w http.ResponseWriter, r *http.Request) { } data.Filter.Project = project + + if !h.A.Licenser.AdvancedWebhookFiltering() { + data.Filter.Query = "" // event payload search not allowed + } + eventsPaged, paginationData, err := postgres.NewEventRepo(h.A.DB, h.A.Cache).LoadEventsPaged(r.Context(), project.UID, data.Filter) if err != nil { log.FromContext(r.Context()).WithError(err).Error("failed to fetch events") diff --git a/api/handlers/event_delivery.go b/api/handlers/event_delivery.go index 7361bd9d86..1b4bf6ff1b 100644 --- a/api/handlers/event_delivery.go +++ b/api/handlers/event_delivery.go @@ -117,6 +117,28 @@ func (h *Handler) BatchRetryEventDelivery(w http.ResponseWriter, r *http.Request return } + authUser := middleware.GetAuthUserFromContext(r.Context()) + if h.IsReqWithPortalLinkToken(authUser) { + portalLink, err := h.retrievePortalLinkFromToken(r) + if err != nil { + _ = render.Render(w, r, util.NewServiceErrResponse(err)) + return + } + + endpointIDs, err := h.getEndpoints(r, portalLink) + if err != nil { + _ = render.Render(w, r, util.NewServiceErrResponse(err)) + return + } + + if len(endpointIDs) == 0 { + _ = render.Render(w, r, util.NewServerResponse("the portal link doesn't contain any endpoints", nil, http.StatusOK)) + return + } + + data.Filter.EndpointIDs = endpointIDs + } + data.Filter.Project = project ep := datastore.Pageable{} if data.Filter.Pageable == ep { diff --git a/api/handlers/license.go b/api/handlers/license.go new file mode 100644 index 0000000000..968764b2e0 --- /dev/null +++ b/api/handlers/license.go @@ -0,0 +1,21 @@ +package handlers + +import ( + "net/http" + + "github.com/frain-dev/convoy/pkg/log" + + "github.com/frain-dev/convoy/util" + "github.com/go-chi/render" +) + +func (h *Handler) GetLicenseFeatures(w http.ResponseWriter, r *http.Request) { + v, err := h.A.Licenser.FeatureListJSON(r.Context()) + if err != nil { + log.FromContext(r.Context()).WithError(err).Error("failed to get license features") + _ = render.Render(w, r, util.NewErrorResponse("failed to get license features", http.StatusBadRequest)) + return + } + + _ = render.Render(w, r, util.NewServerResponse("Retrieved license features successfully", v, http.StatusOK)) +} diff --git a/api/handlers/middleware.go b/api/handlers/middleware.go new file mode 100644 index 0000000000..a314ed7ce9 --- /dev/null +++ b/api/handlers/middleware.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "errors" + "net/http" + + "github.com/frain-dev/convoy/util" + "github.com/go-chi/render" +) + +var ErrProjectDisabled = errors.New("this project has been disabled for write operations until you re-subscribe your convoy instance") + +func (h *Handler) RequireEnabledProject() func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p, err := h.retrieveProject(r) + if err != nil { + _ = render.Render(w, r, util.NewErrorResponse("failed to retrieve project", http.StatusBadRequest)) + return + } + + if !h.A.Licenser.ProjectEnabled(p.UID) { + _ = render.Render(w, r, util.NewErrorResponse(ErrProjectDisabled.Error(), http.StatusBadRequest)) + return + } + + next.ServeHTTP(w, r) + }) + } +} diff --git a/api/handlers/organisation.go b/api/handlers/organisation.go index d95670f8af..5004d0bf16 100644 --- a/api/handlers/organisation.go +++ b/api/handlers/organisation.go @@ -67,6 +67,7 @@ func (h *Handler) CreateOrganisation(w http.ResponseWriter, r *http.Request) { OrgMemberRepo: postgres.NewOrgMemberRepo(h.A.DB, h.A.Cache), NewOrg: &newOrg, User: user, + Licenser: h.A.Licenser, } organisation, err := co.Run(r.Context()) diff --git a/api/handlers/organisation_invite.go b/api/handlers/organisation_invite.go index b205f10772..2e11effcc1 100644 --- a/api/handlers/organisation_invite.go +++ b/api/handlers/organisation_invite.go @@ -47,6 +47,7 @@ func (h *Handler) InviteUserToOrganisation(w http.ResponseWriter, r *http.Reques Queue: h.A.Queue, InviteRepo: postgres.NewOrgInviteRepo(h.A.DB, h.A.Cache), InviteeEmail: newIV.InviteeEmail, + Licenser: h.A.Licenser, Role: newIV.Role, User: user, Organisation: org, @@ -104,6 +105,7 @@ func (h *Handler) ProcessOrganisationMemberInvite(w http.ResponseWriter, r *http UserRepo: postgres.NewUserRepo(h.A.DB, h.A.Cache), OrgRepo: postgres.NewOrgRepo(h.A.DB, h.A.Cache), OrgMemberRepo: postgres.NewOrgMemberRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, Token: token, Accepted: accepted, NewUser: newUser, diff --git a/api/handlers/project.go b/api/handlers/project.go index 032b7f5bb6..402187a46a 100644 --- a/api/handlers/project.go +++ b/api/handlers/project.go @@ -21,9 +21,8 @@ func createProjectService(h *Handler) (*services.ProjectService, error) { projectService, err := services.NewProjectService( apiKeyRepo, projectRepo, eventRepo, - eventDeliveryRepo, h.A.Cache, + eventDeliveryRepo, h.A.Licenser, h.A.Cache, ) - if err != nil { return nil, err } @@ -78,6 +77,8 @@ func (h *Handler) DeleteProject(w http.ResponseWriter, r *http.Request) { return } + h.A.Licenser.RemoveEnabledProject(project.UID) + _ = render.Render(w, r, util.NewServerResponse("Project deleted successfully", nil, http.StatusOK)) } diff --git a/api/handlers/shim.go b/api/handlers/shim.go deleted file mode 100644 index 2fdf6356d2..0000000000 --- a/api/handlers/shim.go +++ /dev/null @@ -1,67 +0,0 @@ -package handlers - -import ( - "fmt" - "net/http" - "strings" - - "github.com/frain-dev/convoy/auth" - "github.com/frain-dev/convoy/internal/pkg/middleware" - "github.com/frain-dev/convoy/util" - "github.com/go-chi/render" -) - -var redirectRoutes = []string{ - "/api/v1/applications", - "/api/v1/events", - "/api/v1/eventdeliveries", - "/api/v1/security", - "/api/v1/subscriptions", - "/api/v1/sources", -} - -func (h *Handler) RedirectToProjects(w http.ResponseWriter, r *http.Request) { - projectID := r.URL.Query().Get("projectID") - - if util.IsStringEmpty(projectID) { - projectID = r.URL.Query().Get("projectID") - } - - if util.IsStringEmpty(projectID) { - authUser := middleware.GetAuthUserFromContext(r.Context()) - - if authUser.Credential.Type == auth.CredentialTypeAPIKey { - projectID = authUser.Role.Project - } - } - - if util.IsStringEmpty(projectID) { - _ = render.Render(w, r, util.NewErrorResponse("projectID query is missing", http.StatusBadRequest)) - return - } - - rElems := strings.Split(r.URL.Path, "/") - - if !(cap(rElems) > 3) { - _ = render.Render(w, r, util.NewErrorResponse("Invalid path", http.StatusBadRequest)) - return - } - - resourcePrefix := strings.Join(rElems[:4], "/") - - if ok := contains(redirectRoutes, resourcePrefix); ok { - forwardedPath := strings.Join(rElems[3:], "/") - redirectURL := fmt.Sprintf("/api/v1/projects/%s/%s?%s", projectID, forwardedPath, r.URL.RawQuery) - - http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) - } -} - -func contains(sl []string, name string) bool { - for _, value := range sl { - if value == name { - return true - } - } - return false -} diff --git a/api/handlers/subscription.go b/api/handlers/subscription.go index 27a6bf18e1..26a83a9a5f 100644 --- a/api/handlers/subscription.go +++ b/api/handlers/subscription.go @@ -216,6 +216,7 @@ func (h *Handler) CreateSubscription(w http.ResponseWriter, r *http.Request) { SubRepo: postgres.NewSubscriptionRepo(h.A.DB, h.A.Cache), EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), SourceRepo: postgres.NewSourceRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, Project: project, NewSubscription: &sub, } @@ -365,6 +366,7 @@ func (h *Handler) UpdateSubscription(w http.ResponseWriter, r *http.Request) { SubRepo: postgres.NewSubscriptionRepo(h.A.DB, h.A.Cache), EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), SourceRepo: postgres.NewSourceRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, ProjectId: project.UID, SubscriptionId: chi.URLParam(r, "subscriptionID"), Update: &update, @@ -400,6 +402,11 @@ func (h *Handler) ToggleSubscriptionStatus(w http.ResponseWriter, r *http.Reques // @Security ApiKeyAuth // @Router /v1/projects/{projectID}/subscriptions/test_filter [post] func (h *Handler) TestSubscriptionFilter(w http.ResponseWriter, r *http.Request) { + if !h.A.Licenser.AdvancedSubscriptions() { + _ = render.Render(w, r, util.NewErrorResponse("your instance does not have access to subscription filters, upgrade to access this feature", http.StatusBadRequest)) + return + } + var test models.TestFilter err := util.ReadJSON(r, &test) if err != nil { @@ -442,6 +449,11 @@ func (h *Handler) TestSubscriptionFilter(w http.ResponseWriter, r *http.Request) // @Security ApiKeyAuth // @Router /v1/projects/{projectID}/subscriptions/test_function [post] func (h *Handler) TestSubscriptionFunction(w http.ResponseWriter, r *http.Request) { + if !h.A.Licenser.Transformations() { + _ = render.Render(w, r, util.NewErrorResponse("your instance does not have access to transformations, upgrade to access this feature", http.StatusBadRequest)) + return + } + var test models.FunctionRequest err := util.ReadJSON(r, &test) if err != nil { diff --git a/api/handlers/user.go b/api/handlers/user.go index 7120594120..b6b33caa2d 100644 --- a/api/handlers/user.go +++ b/api/handlers/user.go @@ -48,8 +48,10 @@ func (h *Handler) RegisterUser(w http.ResponseWriter, r *http.Request) { Queue: h.A.Queue, JWT: jwt.NewJwt(&config.Auth.Jwt, h.A.Cache), ConfigRepo: postgres.NewConfigRepo(h.A.DB), - BaseURL: baseUrl, - Data: &newUser, + Licenser: h.A.Licenser, + + BaseURL: baseUrl, + Data: &newUser, } user, token, err := rs.Run(r.Context()) diff --git a/api/ingest.go b/api/ingest.go index 6cbb90ec69..5af361c2b1 100644 --- a/api/ingest.go +++ b/api/ingest.go @@ -3,15 +3,16 @@ package api import ( "encoding/json" "errors" + "io" + "net/http" "strings" + "time" + + "github.com/frain-dev/convoy/api/handlers" "github.com/frain-dev/convoy/pkg/msgpack" "gopkg.in/guregu/null.v4" - "io" - "net/http" - "time" - "github.com/frain-dev/convoy/internal/pkg/dedup" "github.com/go-chi/chi/v5" "github.com/oklog/ulid/v2" @@ -65,6 +66,11 @@ func (a *ApplicationHandler) IngestEvent(w http.ResponseWriter, r *http.Request) return } + if !a.A.Licenser.ProjectEnabled(project.UID) { + _ = render.Render(w, r, util.NewErrorResponse(handlers.ErrProjectDisabled.Error(), http.StatusBadRequest)) + return + } + if source.Type != datastore.HTTPSource { _ = render.Render(w, r, util.NewErrorResponse("Source type needs to be HTTP", http.StatusBadRequest)) diff --git a/api/ingest_integration_test.go b/api/ingest_integration_test.go index e1aa2cf6ac..09e8f53ac6 100644 --- a/api/ingest_integration_test.go +++ b/api/ingest_integration_test.go @@ -209,6 +209,7 @@ func (i *IngestIntegrationTestSuite) Test_IngestEvent_GoodAPIKey() { // Act. i.Router.ServeHTTP(w, req) + fmt.Println("eee", w.Body.String()) // Assert. require.Equal(i.T(), expectedStatusCode, w.Code) } diff --git a/api/models/project.go b/api/models/project.go index f2f7f86b84..656c1aeaeb 100644 --- a/api/models/project.go +++ b/api/models/project.go @@ -88,6 +88,7 @@ func (pc *ProjectConfig) Transform() *datastore.ProjectConfig { AddEventIDTraceHeaders: pc.AddEventIDTraceHeaders, MultipleEndpointSubscriptions: pc.MultipleEndpointSubscriptions, SSL: pc.SSL.transform(), + SearchPolicy: pc.SearchPolicy, RateLimit: pc.RateLimit.Transform(), Strategy: pc.Strategy.transform(), Signature: pc.Signature.transform(), diff --git a/api/server_suite_test.go b/api/server_suite_test.go index f3a5e25623..ba89a798bb 100644 --- a/api/server_suite_test.go +++ b/api/server_suite_test.go @@ -7,7 +7,6 @@ import ( "bytes" "encoding/json" "fmt" - rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" "io" "math/rand" "net/http" @@ -18,6 +17,9 @@ import ( "testing" "time" + noopLicenser "github.com/frain-dev/convoy/internal/pkg/license/noop" + rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" + ncache "github.com/frain-dev/convoy/cache/noop" "github.com/frain-dev/convoy/api/models" @@ -64,8 +66,10 @@ func getConfig() config.Configuration { return cfg } -var once sync.Once -var pDB *postgres.Postgres +var ( + once sync.Once + pDB *postgres.Postgres +) func getDB() database.Database { once.Do(func() { @@ -125,11 +129,12 @@ func buildServer() *ApplicationHandler { ah, _ := NewApplicationHandler( &types.APIOptions{ - DB: db, - Queue: newQueue, - Logger: logger, - Cache: noopCache, - Rate: r, + DB: db, + Queue: newQueue, + Logger: logger, + Cache: noopCache, + Rate: r, + Licenser: noopLicenser.NewLicenser(), }) _ = ah.RegisterPolicy() diff --git a/api/types/types.go b/api/types/types.go index 539c67289c..6b706f2f14 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -5,6 +5,7 @@ import ( "github.com/frain-dev/convoy/cache" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/queue" @@ -13,11 +14,12 @@ import ( type ContextKey string type APIOptions struct { - FFlag *fflag.FFlag - DB database.Database - Queue queue.Queuer - Logger log.StdLogger - Cache cache.Cache - Authz *authz.Authz - Rate limiter.RateLimiter + FFlag *fflag.FFlag + DB database.Database + Queue queue.Queuer + Logger log.StdLogger + Cache cache.Cache + Authz *authz.Authz + Rate limiter.RateLimiter + Licenser license.Licenser } diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index d9f1dfc76a..0628cb7f78 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -3,12 +3,13 @@ package agent import ( "context" "fmt" - workerSrv "github.com/frain-dev/convoy/cmd/worker" - "github.com/frain-dev/convoy/util" "os" "os/signal" "time" + workerSrv "github.com/frain-dev/convoy/cmd/worker" + "github.com/frain-dev/convoy/util" + "github.com/frain-dev/convoy/api" "github.com/frain-dev/convoy/api/types" "github.com/frain-dev/convoy/auth/realm_chain" @@ -138,7 +139,7 @@ func startServerComponent(ctx context.Context, a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag() + flag, err := fflag.NewFFlag(&cfg) if err != nil { a.Logger.WithError(err).Fatal("failed to create fflag controller") } @@ -157,12 +158,13 @@ func startServerComponent(ctx context.Context, a *cli.App) error { evHandler, err := api.NewApplicationHandler( &types.APIOptions{ - FFlag: flag, - DB: a.DB, - Queue: a.Queue, - Logger: lo, - Cache: a.Cache, - Rate: a.Rate, + FFlag: flag, + DB: a.DB, + Queue: a.Queue, + Logger: lo, + Cache: a.Cache, + Rate: a.Rate, + Licenser: a.Licenser, }) if err != nil { return err diff --git a/cmd/bootstrap/bootstrap.go b/cmd/bootstrap/bootstrap.go index 1d4b1f60fa..1c4f75614d 100644 --- a/cmd/bootstrap/bootstrap.go +++ b/cmd/bootstrap/bootstrap.go @@ -33,6 +33,15 @@ func AddBootstrapCommand(a *cli.App) *cobra.Command { "ShouldBootstrap": "false", }, RunE: func(cmd *cobra.Command, args []string) error { + ok, err := a.Licenser.CreateUser(context.Background()) + if err != nil { + return err + } + + if !ok { + return services.ErrUserLimit + } + if format != "json" && format != "human" { return errors.New("unsupported output format") } diff --git a/cmd/ff/feature_flags.go b/cmd/ff/feature_flags.go new file mode 100644 index 0000000000..af9dc6b5f2 --- /dev/null +++ b/cmd/ff/feature_flags.go @@ -0,0 +1,33 @@ +package ff + +import ( + "github.com/frain-dev/convoy/config" + fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/pkg/log" + "github.com/spf13/cobra" +) + +func AddFeatureFlagsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "feature-flags", + Short: "Print the list of feature flags", + Annotations: map[string]string{ + "CheckMigration": "true", + "ShouldBootstrap": "false", + }, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Get() + if err != nil { + log.WithError(err).Fatalf("Error fetching the config.") + } + f, err := fflag2.NewFFlag(&cfg) + if err != nil { + return err + } + return f.ListFeatures() + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) {}, + } + + return cmd +} diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index a370d57265..ad46e74162 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -9,6 +9,9 @@ import ( "os" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/internal/pkg/license/keygen" + "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/util" @@ -64,6 +67,17 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ return err } + postgresDB, err := postgres.NewDB(cfg) + if err != nil { + return err + } + + *db = *postgresDB + + if _, ok := skipHook[cmd.Use]; ok { + return nil + } + cfg, err = config.Get() // updated if err != nil { return err @@ -117,13 +131,6 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ return err } - postgresDB, err := postgres.NewDB(cfg) - if err != nil { - return err - } - - *db = *postgresDB - hooks := dbhook.Init() // the order matters here @@ -182,6 +189,26 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ app.Rate = rateLimiter + app.Licenser, err = license.NewLicenser(&license.Config{ + KeyGen: keygen.Config{ + LicenseKey: cfg.LicenseKey, + OrgRepo: postgres.NewOrgRepo(app.DB, app.Cache), + UserRepo: postgres.NewUserRepo(app.DB, app.Cache), + ProjectRepo: projectRepo, + }, + }) + if err != nil { + return err + } + + if !app.Licenser.ConsumerPoolTuning() { + cfg.ConsumerPoolSize = config.DefaultConfiguration.ConsumerPoolSize + } + + if err = config.Override(&cfg); err != nil { + return err + } + // update config singleton with the instance id if _, ok := skipConfigLoadCmd[cmd.Use]; !ok { configRepo := postgres.NewConfigRepo(app.DB) @@ -203,8 +230,16 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ // these commands don't need to load instance config var skipConfigLoadCmd = map[string]struct{}{ "bootstrap": {}, - "version": {}, - "migrate": {}, +} + +// commands dont need the hooks +var skipHook = map[string]struct{}{ + // migrate commands + "up": {}, + "down": {}, + "create": {}, + + "version": {}, } func PostRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args []string) error { @@ -333,6 +368,14 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { c := &config.Configuration{} + // CONVOY_LICENSE_KEY + licenseKey, err := cmd.Flags().GetString("license-key") + if err != nil { + return nil, err + } + + c.LicenseKey = licenseKey + // CONVOY_DB_TYPE dbType, err := cmd.Flags().GetString("db-type") if err != nil { @@ -452,15 +495,11 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { } // Feature flags - fflag, err := cmd.Flags().GetString("feature-flag") + fflag, err := cmd.Flags().GetStringSlice("enable-feature-flag") if err != nil { return nil, err } - - switch fflag { - case config.Experimental: - c.FeatureFlag = config.ExperimentalFlagLevel - } + c.EnableFeatureFlag = fflag // tracing tracingProvider, err := cmd.Flags().GetString("tracer-type") @@ -521,14 +560,14 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { } - flag, err := fflag2.NewFFlag() + flag, err := fflag2.NewFFlag(c) if err != nil { return nil, err } c.Metrics = config.MetricsConfiguration{ IsEnabled: false, } - if flag.CanAccessFeature(fflag2.Prometheus, c) { + if flag.CanAccessFeature(fflag2.Prometheus) { metricsBackend, err := cmd.Flags().GetString("metrics-backend") if err != nil { return nil, err @@ -654,14 +693,13 @@ func shouldBootstrap(cmd *cobra.Command) bool { } func ensureDefaultUser(ctx context.Context, a *cli.App) error { - pageable := datastore.Pageable{PerPage: 10, Direction: datastore.Next, NextCursor: datastore.DefaultCursor} userRepo := postgres.NewUserRepo(a.DB, a.Cache) - users, _, err := userRepo.LoadUsersPaged(ctx, pageable) + count, err := userRepo.CountUsers(ctx) if err != nil { - return fmt.Errorf("failed to load users - %w", err) + return fmt.Errorf("failed to count users: %v", err) } - if len(users) > 0 { + if count > 0 { return nil } diff --git a/cmd/ingest/ingest.go b/cmd/ingest/ingest.go index d52cf83d16..334f7bdbd1 100644 --- a/cmd/ingest/ingest.go +++ b/cmd/ingest/ingest.go @@ -3,6 +3,7 @@ package ingest import ( "context" "fmt" + "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/internal/pkg/cli" @@ -111,7 +112,7 @@ func StartIngest(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - ingest, err := pubsub.NewIngest(ctx, sourceTable, a.Queue, lo, rateLimiter, host) + ingest, err := pubsub.NewIngest(ctx, sourceTable, a.Queue, lo, rateLimiter, a.Licenser, host) if err != nil { return err } diff --git a/cmd/main.go b/cmd/main.go index 17f1e9f361..d30320bad4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/frain-dev/convoy/cmd/ff" "os" _ "time/tzdata" @@ -47,7 +48,7 @@ func main() { var dbPassword string var dbDatabase string - var fflag string + var fflag []string var enableProfiling bool var redisPort int @@ -72,9 +73,12 @@ func main() { var maxRetrySeconds uint64 + var licenseKey string + var configFile string c.Flags().StringVar(&configFile, "config", "./convoy.json", "Configuration file for convoy") + c.Flags().StringVar(&licenseKey, "license-key", "", "Convoy license key") // db config c.Flags().StringVar(&dbHost, "db-host", "", "Database Host") @@ -96,8 +100,7 @@ func main() { c.Flags().StringVar(&redisDatabase, "redis-database", "", "Redis database") c.Flags().IntVar(&redisPort, "redis-port", 0, "Redis Port") - c.Flags().StringVar(&fflag, "feature-flag", "", "Enable feature flags (experimental)") - + c.Flags().StringSliceVar(&fflag, "enable-feature-flag", []string{}, "List of feature flags to enable e.g. \"full-text-search,prometheus\"") // tracing c.Flags().StringVar(&tracerType, "tracer-type", "", "Tracer backend, e.g. sentry, datadog or otel") c.Flags().StringVar(&sentryDSN, "sentry-dsn", "", "Sentry backend dsn") @@ -107,7 +110,7 @@ func main() { c.Flags().StringVar(&otelAuthHeaderValue, "otel-auth-header-value", "", "OTel backend auth header value") // metrics - c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('experimental' feature flag level required") + c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('prometheus' feature flag required") c.Flags().Uint64Var(&prometheusMetricsSampleTime, "metrics-prometheus-sample-time", 5, "Prometheus metrics sample time") c.Flags().StringVar(&retentionPolicy, "retention-policy", "", "Retention Policy Duration") @@ -128,6 +131,7 @@ func main() { c.AddCommand(ingest.AddIngestCommand(app)) c.AddCommand(bootstrap.AddBootstrapCommand(app)) c.AddCommand(agent.AddAgentCommand(app)) + c.AddCommand(ff.AddFeatureFlagsCommand()) if err := c.Execute(); err != nil { slog.Fatal(err) diff --git a/cmd/server/server.go b/cmd/server/server.go index b397a96572..87d9be3486 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -106,7 +106,7 @@ func startConvoyServer(a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag() + flag, err := fflag.NewFFlag(&cfg) if err != nil { a.Logger.WithError(err).Fatal("failed to create fflag controller") } @@ -128,12 +128,13 @@ func startConvoyServer(a *cli.App) error { handler, err := api.NewApplicationHandler( &types.APIOptions{ - FFlag: flag, - DB: a.DB, - Queue: a.Queue, - Logger: lo, - Cache: a.Cache, - Rate: a.Rate, + FFlag: flag, + DB: a.DB, + Queue: a.Queue, + Logger: lo, + Cache: a.Cache, + Rate: a.Rate, + Licenser: a.Licenser, }) if err != nil { return err diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index ddf0ec38cd..3b427ec225 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -3,10 +3,13 @@ package worker import ( "context" "fmt" + "net/http" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/internal/pkg/cli" + fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/loader" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -28,7 +31,6 @@ import ( "github.com/go-chi/render" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" - "net/http" ) func AddWorkerCommand(a *cli.App) *cobra.Command { @@ -238,7 +240,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte telemetry.OptionBackend(pb), telemetry.OptionBackend(mb)) - dispatcher, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, false) + dispatcher, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, a.Licenser, false) if err != nil { a.Logger.WithError(err).Fatal("Failed to create new net dispatcher") return err @@ -257,6 +259,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, eventDeliveryRepo, + a.Licenser, projectRepo, a.Queue, rateLimiter, @@ -272,7 +275,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte eventDeliveryRepo, a.Queue, subRepo, - deviceRepo), newTelemetry) + deviceRepo, a.Licenser), newTelemetry) consumer.RegisterHandlers(convoy.RetryEventProcessor, task.ProcessRetryEventDelivery( endpointRepo, @@ -293,6 +296,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte a.Queue, subRepo, deviceRepo, + a.Licenser, subscriptionsTable), newTelemetry) consumer.RegisterHandlers(convoy.CreateDynamicEventProcessor, task.ProcessDynamicEventCreation( @@ -302,7 +306,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte eventDeliveryRepo, a.Queue, subRepo, - deviceRepo), newTelemetry) + deviceRepo, a.Licenser), newTelemetry) consumer.RegisterHandlers(convoy.RetentionPolicies, task.RetentionPolicies( configRepo, @@ -320,11 +324,17 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.DailyAnalytics, task.PushDailyTelemetry(lo, a.DB, a.Cache, rd), nil) consumer.RegisterHandlers(convoy.EmailProcessor, task.ProcessEmails(sc), nil) - consumer.RegisterHandlers(convoy.TokenizeSearch, task.GeneralTokenizerHandler(projectRepo, eventRepo, jobRepo, rd), nil) - consumer.RegisterHandlers(convoy.TokenizeSearchForProject, task.TokenizerHandler(eventRepo, jobRepo), nil) + fflag, err := fflag2.NewFFlag(&cfg) + if err != nil { + return nil + } + if fflag.CanAccessFeature(fflag2.FullTextSearch) && a.Licenser.AdvancedWebhookFiltering() { + consumer.RegisterHandlers(convoy.TokenizeSearch, task.GeneralTokenizerHandler(projectRepo, eventRepo, jobRepo, rd), nil) + consumer.RegisterHandlers(convoy.TokenizeSearchForProject, task.TokenizerHandler(eventRepo, jobRepo), nil) + } consumer.RegisterHandlers(convoy.NotificationProcessor, task.ProcessNotifications(sc), nil) - consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo), nil) + consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher), nil) consumer.RegisterHandlers(convoy.DeleteArchivedTasksProcessor, task.DeleteArchivedTasks(a.Queue, rd), nil) metrics.RegisterQueueMetrics(a.Queue, a.DB) diff --git a/config/config.go b/config/config.go index 6f2ebebe18..d61082e71c 100644 --- a/config/config.go +++ b/config/config.go @@ -307,7 +307,7 @@ type OnPremStorage struct { } type MetricsConfiguration struct { - IsEnabled bool `json:"metrics_enabled" envconfig:"CONVOY_METRICS_ENABLED"` + IsEnabled bool `json:"enabled" envconfig:"CONVOY_METRICS_ENABLED"` Backend MetricsBackend `json:"metrics_backend" envconfig:"CONVOY_METRICS_BACKEND"` Prometheus PrometheusMetricsConfiguration `json:"prometheus_metrics"` } @@ -345,7 +345,6 @@ type ( LimiterProvider string DatabaseProvider string SearchProvider string - FeatureFlagProvider string MetricsBackend string ) @@ -353,31 +352,6 @@ func (s SignatureHeaderProvider) String() string { return string(s) } -type FlagLevel int - -const ( - ExperimentalFlagLevel FlagLevel = iota + 1 -) - -const Experimental = "experimental" - -func (ft *FlagLevel) UnmarshalJSON(v []byte) error { - switch string(v) { - case Experimental: - *ft = ExperimentalFlagLevel - } - return nil -} - -func (ft FlagLevel) MarshalJSON() ([]byte, error) { - switch ft { - case ExperimentalFlagLevel: - return []byte(fmt.Sprintf(`"%s"`, []byte(Experimental))), nil - default: - return []byte(fmt.Sprintf(`"%s"`, []byte(Experimental))), nil - } -} - type ExecutionMode string const ( @@ -402,7 +376,7 @@ type Configuration struct { Host string `json:"host" envconfig:"CONVOY_HOST"` Pyroscope PyroscopeConfiguration `json:"pyroscope"` CustomDomainSuffix string `json:"custom_domain_suffix" envconfig:"CONVOY_CUSTOM_DOMAIN_SUFFIX"` - FeatureFlag FlagLevel `json:"feature_flag" envconfig:"CONVOY_FEATURE_FLAG"` + EnableFeatureFlag []string `json:"enable_feature_flag" envconfig:"CONVOY_ENABLE_FEATURE_FLAG"` RetentionPolicy RetentionPolicyConfiguration `json:"retention_policy"` CircuitBreaker CircuitBreakerConfiguration `json:"circuit_breaker"` Analytics AnalyticsConfiguration `json:"analytics"` @@ -413,6 +387,7 @@ type Configuration struct { InstanceIngestRate int `json:"instance_ingest_rate" envconfig:"CONVOY_INSTANCE_INGEST_RATE"` WorkerExecutionMode ExecutionMode `json:"worker_execution_mode" envconfig:"CONVOY_WORKER_EXECUTION_MODE"` MaxRetrySeconds uint64 `json:"max_retry_seconds,omitempty" envconfig:"CONVOY_MAX_RETRY_SECONDS"` + LicenseKey string `json:"license_key" envconfig:"CONVOY_LICENSE_KEY"` } type PyroscopeConfiguration struct { diff --git a/configs/convoy.templ.json b/configs/convoy.templ.json index 20ea2d4155..d9fc65086f 100644 --- a/configs/convoy.templ.json +++ b/configs/convoy.templ.json @@ -26,13 +26,6 @@ "password": "", "from": "support@frain.dev" }, - "search": { - "type": "typesense", - "typesense": { - "host": "http://typesense:8108", - "api_key": "convoy" - } - }, "server": { "http": { "ssl": false, diff --git a/configs/docker-compose.templ.yml b/configs/docker-compose.templ.yml index c7604b17d7..b3787fc020 100644 --- a/configs/docker-compose.templ.yml +++ b/configs/docker-compose.templ.yml @@ -2,7 +2,7 @@ version: "3" services: web: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: [ "/start.sh" ] hostname: web container_name: web @@ -12,12 +12,11 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster scheduler: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: ["./cmd", "scheduler", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -25,12 +24,11 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster worker: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: ["./cmd", "worker", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -38,12 +36,11 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster ingest: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: ["./cmd", "ingest", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -51,7 +48,6 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster @@ -76,20 +72,6 @@ services: networks: - backendCluster - typesense: - image: typesense/typesense:0.22.2 - hostname: typesense - container_name: typesense - restart: always - environment: - TYPESENSE_DATA_DIR: /data/typesense - TYPESENSE_ENABLE_CORS: "true" - TYPESENSE_API_KEY: "convoy" - volumes: - - ./typesense-data:/data/typesense - networks: - - backendCluster - caddy: image: caddy restart: unless-stopped diff --git a/configs/local/docker-compose.yml b/configs/local/docker-compose.yml index b84bcbd0b2..5c9085b255 100644 --- a/configs/local/docker-compose.yml +++ b/configs/local/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: web: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.5.1 + image: getconvoy/convoy:latest command: ["/start.sh"] volumes: - ./convoy.json:/convoy.json @@ -19,9 +19,8 @@ services: - postgres - redis_server - pgbouncer - worker: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.5.1 + image: getconvoy/convoy:latest command: ["./cmd", "worker", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -29,9 +28,8 @@ services: depends_on: web: condition: service_healthy - ingest: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.5.1 + image: getconvoy/convoy:latest command: ["./cmd", "ingest", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -39,13 +37,13 @@ services: depends_on: web: condition: service_healthy - pgbouncer: image: bitnami/pgbouncer:latest hostname: pgbouncer restart: unless-stopped depends_on: - - postgres + postgres: + condition: service_healthy env_file: - ./conf/.env volumes: @@ -53,10 +51,9 @@ services: - ./conf/userlists.txt:/bitnami/userlists.txt ports: - "6432:6432" - postgres: image: bitnami/postgresql:latest - restart: unless-stopped + restart: on-failure ports: - "5432:5432" environment: @@ -68,13 +65,17 @@ services: POSTGRESQL_SHARED_PRELOAD_LIBRARIES: pg_stat_statements volumes: - postgresql_master_data:/bitnami/postgresql - + healthcheck: + test: ["CMD-SHELL", "pg_isready -U convoy"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s redis_server: image: redis:alpine restart: unless-stopped volumes: - redis_data:/data - volumes: postgresql_master_data: redis_data: diff --git a/database/postgres/event_delivery.go b/database/postgres/event_delivery.go index e138385374..2d82aa6b88 100644 --- a/database/postgres/event_delivery.go +++ b/database/postgres/event_delivery.go @@ -61,6 +61,8 @@ const ( COALESCE(ep.project_id, '') AS "endpoint_metadata.project_id", COALESCE(ep.support_email, '') AS "endpoint_metadata.support_email", COALESCE(ep.url, '') AS "endpoint_metadata.url", + COALESCE(ep.owner_id, '') AS "endpoint_metadata.owner_id", + ev.id AS "event_metadata.id", ev.event_type AS "event_metadata.event_type", COALESCE(ed.latency_seconds, 0) AS latency_seconds, @@ -738,6 +740,7 @@ func (e *eventDeliveryRepo) LoadEventDeliveriesPaged(ctx context.Context, projec Url: ev.Endpoint.URL.ValueOrZero(), Name: ev.Endpoint.Name.ValueOrZero(), SupportEmail: ev.Endpoint.SupportEmail.ValueOrZero(), + OwnerID: ev.Endpoint.OwnerID.ValueOrZero(), }, Source: &datastore.Source{ UID: ev.Source.UID.ValueOrZero(), @@ -945,6 +948,7 @@ type EndpointMetadata struct { URL null.String `db:"url"` ProjectID null.String `db:"project_id"` SupportEmail null.String `db:"support_email"` + OwnerID null.String `db:"owner_id"` } type EventMetadata struct { diff --git a/database/postgres/organisation.go b/database/postgres/organisation.go index d35881b913..14a13b771a 100644 --- a/database/postgres/organisation.go +++ b/database/postgres/organisation.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/cache" ncache "github.com/frain-dev/convoy/cache/noop" @@ -79,6 +80,11 @@ const ( GROUP BY id ORDER BY id DESC LIMIT 1` + + countOrganizations = ` + SELECT COUNT(*) AS count + FROM convoy.organisations + WHERE deleted_at IS NULL` ) type orgRepo struct { @@ -254,6 +260,16 @@ func (o *orgRepo) DeleteOrganisation(ctx context.Context, uid string) error { return nil } +func (o *orgRepo) CountOrganisations(ctx context.Context) (int64, error) { + var count int64 + err := o.db.GetContext(ctx, &count, countOrganizations) + if err != nil { + return 0, err + } + + return count, nil +} + func (o *orgRepo) FetchOrganisationByID(ctx context.Context, id string) (*datastore.Organisation, error) { fromCache, err := o.readFromCache(ctx, id, func() (*datastore.Organisation, error) { org := &datastore.Organisation{} @@ -267,7 +283,6 @@ func (o *orgRepo) FetchOrganisationByID(ctx context.Context, id string) (*datast return org, nil }) - if err != nil { return nil, err } @@ -288,7 +303,6 @@ func (o *orgRepo) FetchOrganisationByAssignedDomain(ctx context.Context, domain return org, nil }) - if err != nil { return nil, err } @@ -309,7 +323,6 @@ func (o *orgRepo) FetchOrganisationByCustomDomain(ctx context.Context, domain st return org, nil }) - if err != nil { return nil, err } diff --git a/database/postgres/organisation_member.go b/database/postgres/organisation_member.go index 2c5670d207..259c91477c 100644 --- a/database/postgres/organisation_member.go +++ b/database/postgres/organisation_member.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/frain-dev/convoy/cache" "github.com/frain-dev/convoy/util" diff --git a/database/postgres/organisation_test.go b/database/postgres/organisation_test.go index 26a9c2ac15..4cbdaae797 100644 --- a/database/postgres/organisation_test.go +++ b/database/postgres/organisation_test.go @@ -47,6 +47,33 @@ func TestLoadOrganisationsPaged(t *testing.T) { require.Equal(t, 2, len(organisations)) } +func TestCountOrganisations(t *testing.T) { + db, closeFn := getDB(t) + defer closeFn() + + orgRepo := NewOrgRepo(db, nil) + + user := seedUser(t, db) + count := 10 + for i := 0; i < count; i++ { + org := &datastore.Organisation{ + UID: ulid.Make().String(), + OwnerID: user.UID, + Name: fmt.Sprintf("org%d", i), + CustomDomain: null.NewString(ulid.Make().String(), true), + AssignedDomain: null.NewString(ulid.Make().String(), true), + } + + err := orgRepo.CreateOrganisation(context.Background(), org) + require.NoError(t, err) + } + + orgCount, err := orgRepo.CountOrganisations(context.Background()) + + require.NoError(t, err) + require.Equal(t, int64(count), orgCount) +} + func TestCreateOrganisation(t *testing.T) { db, closeFn := getDB(t) defer closeFn() diff --git a/database/postgres/project.go b/database/postgres/project.go index 92e87ebaea..8c8d860ea9 100644 --- a/database/postgres/project.go +++ b/database/postgres/project.go @@ -111,8 +111,7 @@ const ( LEFT JOIN convoy.project_configurations c ON p.project_configuration_id = c.id WHERE p.id = $1 AND p.deleted_at IS NULL; - ` - +` fetchProjects = ` SELECT p.id, @@ -210,6 +209,11 @@ const ( GROUP BY p.id ORDER BY events_count DESC; ` + + countProjects = ` + SELECT COUNT(*) AS count + FROM convoy.projects + WHERE deleted_at IS NULL` ) type projectRepo struct { @@ -225,6 +229,16 @@ func NewProjectRepo(db database.Database, ca cache.Cache) datastore.ProjectRepos return &projectRepo{db: db.GetDB(), hook: db.GetHook(), cache: ca} } +func (o *projectRepo) CountProjects(ctx context.Context) (int64, error) { + var count int64 + err := o.db.GetContext(ctx, &count, countProjects) + if err != nil { + return 0, err + } + + return count, nil +} + func (p *projectRepo) CreateProject(ctx context.Context, project *datastore.Project) error { tx, err := p.db.BeginTxx(ctx, &sql.TxOptions{}) if err != nil { diff --git a/database/postgres/project_test.go b/database/postgres/project_test.go index 67c31a4ed1..64278e2b64 100644 --- a/database/postgres/project_test.go +++ b/database/postgres/project_test.go @@ -61,6 +61,32 @@ func Test_FetchProjectByID(t *testing.T) { require.Equal(t, newProject, dbProject) } +func TestCountProjects(t *testing.T) { + db, closeFn := getDB(t) + defer closeFn() + + projectRepository := NewProjectRepo(db, nil) + org := seedOrg(t, db) + + count := 10 + for i := 0; i < count; i++ { + project := &datastore.Project{ + UID: ulid.Make().String(), + Name: ulid.Make().String(), + OrganisationID: org.UID, + Type: datastore.IncomingProject, + Config: &datastore.DefaultProjectConfig, + } + + err := projectRepository.CreateProject(context.Background(), project) + require.NoError(t, err) + } + + projectCount, err := projectRepository.CountProjects(context.Background()) + require.NoError(t, err) + require.Equal(t, int64(count), projectCount) +} + func Test_CreateProject(t *testing.T) { db, closeFn := getDB(t) defer closeFn() diff --git a/database/postgres/users.go b/database/postgres/users.go index e18b36b1a4..f4e89eb450 100644 --- a/database/postgres/users.go +++ b/database/postgres/users.go @@ -41,35 +41,10 @@ const ( WHERE deleted_at IS NULL ` - fetchUsersPaginated = ` - SELECT * FROM convoy.users WHERE deleted_at IS NULL` - - fetchUsersPagedForward = ` - %s - AND id <= :cursor - GROUP BY id - ORDER BY id DESC - LIMIT :limit` - - fetchUsersPagedBackward = ` - WITH users AS ( - %s - AND id >= :cursor - GROUP BY id - ORDER BY id ASC - LIMIT :limit - ) - - SELECT * FROM users ORDER BY id DESC` - - countPrevUsers = ` - SELECT COUNT(DISTINCT(id)) AS count + countUsers = ` + SELECT COUNT(*) AS count FROM convoy.users - WHERE deleted_at IS NULL - AND id > :cursor - GROUP BY id - ORDER BY id DESC - LIMIT 1` + WHERE deleted_at IS NULL` ) var ( @@ -191,91 +166,12 @@ func (u *userRepo) FindUserByEmailVerificationToken(ctx context.Context, token s return user, nil } -func (u *userRepo) LoadUsersPaged(ctx context.Context, pageable datastore.Pageable) ([]datastore.User, datastore.PaginationData, error) { - arg := map[string]interface{}{ - "limit": pageable.Limit(), - "cursor": pageable.Cursor(), - } - - var query string - if pageable.Direction == datastore.Next { - query = fetchUsersPagedForward - } else { - query = fetchUsersPagedBackward - } - - query = fmt.Sprintf(query, fetchUsersPaginated) - - query, args, err := sqlx.Named(query, arg) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - query, args, err = sqlx.In(query, args...) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - query = u.db.Rebind(query) - - rows, err := u.db.QueryxContext(ctx, query, args...) +func (o *userRepo) CountUsers(ctx context.Context) (int64, error) { + var count int64 + err := o.db.GetContext(ctx, &count, countUsers) if err != nil { - return nil, datastore.PaginationData{}, err + return 0, err } - defer closeWithError(rows) - - var users []datastore.User - for rows.Next() { - var user datastore.User - err = rows.StructScan(&user) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - users = append(users, user) - } - - var count datastore.PrevRowCount - if len(users) > 0 { - var countQuery string - var qargs []interface{} - first := users[0] - qarg := arg - qarg["cursor"] = first.UID - - countQuery, qargs, err = sqlx.Named(countPrevUsers, qarg) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - countQuery = u.db.Rebind(countQuery) - - // count the row number before the first row - rows, err := u.db.QueryxContext(ctx, countQuery, qargs...) - if err != nil { - return nil, datastore.PaginationData{}, err - } - defer closeWithError(rows) - - if rows.Next() { - err = rows.StructScan(&count) - if err != nil { - return nil, datastore.PaginationData{}, err - } - } - } - - ids := make([]string, len(users)) - for i := range users { - ids[i] = users[i].UID - } - - if len(users) > pageable.PerPage { - users = users[:len(users)-1] - } - - pagination := &datastore.PaginationData{PrevRowCount: count} - pagination = pagination.Build(pageable, ids) - return users, *pagination, nil + return count, nil } diff --git a/database/postgres/users_test.go b/database/postgres/users_test.go index a020ea6d5c..276bad6f99 100644 --- a/database/postgres/users_test.go +++ b/database/postgres/users_test.go @@ -87,6 +87,31 @@ func Test_CreateUser(t *testing.T) { } } +func TestCountUsers(t *testing.T) { + db, closeFn := getDB(t) + defer closeFn() + + userRepo := NewUserRepo(db, nil) + count := 10 + + for i := 0; i < count; i++ { + u := &datastore.User{ + UID: ulid.Make().String(), + FirstName: "test", + LastName: "test", + Email: fmt.Sprintf("%s@test.com", ulid.Make().String()), + } + + err := userRepo.CreateUser(context.Background(), u) + require.NoError(t, err) + } + + userCount, err := userRepo.CountUsers(context.Background()) + + require.NoError(t, err) + require.Equal(t, int64(count), userCount) +} + func Test_FindUserByEmail(t *testing.T) { db, closeFn := getDB(t) defer closeFn() @@ -210,76 +235,6 @@ func Test_FindUserByEmailVerificationToken(t *testing.T) { require.Equal(t, user, newUser) } -func Test_LoadUsersPaged(t *testing.T) { - type Expected struct { - paginationData datastore.PaginationData - } - - tests := []struct { - name string - pageData datastore.Pageable - count int - expected Expected - }{ - { - name: "Load Users Paged - 10 records", - pageData: datastore.Pageable{PerPage: 3}, - count: 10, - expected: Expected{ - paginationData: datastore.PaginationData{ - PerPage: 3, - }, - }, - }, - - { - name: "Load Users Paged - 12 records", - pageData: datastore.Pageable{PerPage: 4}, - count: 12, - expected: Expected{ - paginationData: datastore.PaginationData{ - PerPage: 4, - }, - }, - }, - - { - name: "Load Users Paged - 5 records", - pageData: datastore.Pageable{PerPage: 3}, - count: 5, - expected: Expected{ - paginationData: datastore.PaginationData{ - PerPage: 3, - }, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - db, closeFn := getDB(t) - defer closeFn() - - userRepo := NewUserRepo(db, nil) - - for i := 0; i < tc.count; i++ { - user := &datastore.User{ - UID: ulid.Make().String(), - FirstName: "test", - LastName: "test", - Email: fmt.Sprintf("%s@test.com", ulid.Make().String()), - } - require.NoError(t, userRepo.CreateUser(context.Background(), user)) - } - - _, pageable, err := userRepo.LoadUsersPaged(context.Background(), tc.pageData) - - require.NoError(t, err) - require.Equal(t, tc.expected.paginationData.PerPage, pageable.PerPage) - }) - } -} - func Test_UpdateUser(t *testing.T) { db, closeFn := getDB(t) defer closeFn() diff --git a/datastore/filter.go b/datastore/filter.go index e50132a0ed..9bcb4c860b 100644 --- a/datastore/filter.go +++ b/datastore/filter.go @@ -46,10 +46,11 @@ type FilterBy struct { SearchParams SearchParams } -func (f *FilterBy) String() *string { +func (f *FilterBy) String() string { var s string filterByBuilder := new(strings.Builder) - filterByBuilder.WriteString(fmt.Sprintf("project_id:=%s", f.ProjectID)) // TODO(daniel, RT): how to work around this? + // TODO(daniel, raymond): how to work around this? + filterByBuilder.WriteString(fmt.Sprintf("project_id:=%s", f.ProjectID)) filterByBuilder.WriteString(fmt.Sprintf(" && created_at:[%d..%d]", f.SearchParams.CreatedAtStart, f.SearchParams.CreatedAtEnd)) if len(f.EndpointID) > 0 { @@ -62,9 +63,7 @@ func (f *FilterBy) String() *string { s = filterByBuilder.String() - // we only return a pointer address here - // because the typesense lib needs a string pointer - return &s + return s } type SearchFilter struct { diff --git a/datastore/filter_test.go b/datastore/filter_test.go index 867c798a73..d7c6f034d9 100644 --- a/datastore/filter_test.go +++ b/datastore/filter_test.go @@ -42,7 +42,7 @@ func Test_FilterBy(t *testing.T) { for _, tt := range args { t.Run(tt.name, func(t *testing.T) { s := tt.filter.String() - require.Equal(t, tt.expected, *s) + require.Equal(t, tt.expected, s) }) } } diff --git a/datastore/repository.go b/datastore/repository.go index bfdc296528..13be810090 100644 --- a/datastore/repository.go +++ b/datastore/repository.go @@ -58,6 +58,7 @@ type EventRepository interface { type ProjectRepository interface { LoadProjects(context.Context, *ProjectFilter) ([]*Project, error) CreateProject(context.Context, *Project) error + CountProjects(ctx context.Context) (int64, error) UpdateProject(context.Context, *Project) error DeleteProject(ctx context.Context, uid string) error FetchProjectByID(context.Context, string) (*Project, error) @@ -67,6 +68,7 @@ type ProjectRepository interface { type OrganisationRepository interface { LoadOrganisationsPaged(context.Context, Pageable) ([]Organisation, PaginationData, error) + CountOrganisations(ctx context.Context) (int64, error) CreateOrganisation(context.Context, *Organisation) error UpdateOrganisation(context.Context, *Organisation) error DeleteOrganisation(context.Context, string) error @@ -164,11 +166,11 @@ type JobRepository interface { type UserRepository interface { CreateUser(context.Context, *User) error UpdateUser(ctx context.Context, user *User) error + CountUsers(ctx context.Context) (int64, error) FindUserByEmail(context.Context, string) (*User, error) FindUserByID(context.Context, string) (*User, error) FindUserByToken(context.Context, string) (*User, error) FindUserByEmailVerificationToken(ctx context.Context, token string) (*User, error) - LoadUsersPaged(context.Context, Pageable) ([]User, PaginationData, error) } type ConfigurationRepository interface { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 41a54390b7..2f186d8a68 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,7 +3,6 @@ version: "3" volumes: postgres_data: redis_data: - typesense_data: services: web: @@ -19,7 +18,6 @@ services: depends_on: - postgres - redis_server - - typesense scheduler: build: @@ -32,7 +30,6 @@ services: depends_on: - postgres - redis_server - - typesense worker: build: @@ -45,7 +42,6 @@ services: depends_on: - postgres - redis_server - - typesense ingest: build: @@ -58,7 +54,6 @@ services: depends_on: - postgres - redis_server - - typesense postgres: image: postgres:15.2-alpine @@ -77,16 +72,6 @@ services: volumes: - ./redis_data:/data - typesense: - image: typesense/typesense:0.22.2 - restart: always - environment: - TYPESENSE_DATA_DIR: /data/typesense - TYPESENSE_ENABLE_CORS: "true" - TYPESENSE_API_KEY: "convoy" - volumes: - - ./typesense_data:/data/typesense - prometheus: image: prom/prometheus:v2.24.0 volumes: diff --git a/docs/docs.go b/docs/docs.go index 63f96d0ee0..0a4007ee3f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1,4 +1,4 @@ -// Package docs Code generated by swaggo/swag at 2024-07-10 21:43:37.804513 -0700 PDT m=+1.709850626. DO NOT EDIT +// Package docs Code generated by swaggo/swag at 2024-09-02 15:03:52.730374 +0100 BST m=+2.026310793. DO NOT EDIT package docs import "github.com/swaggo/swag" @@ -5560,6 +5560,9 @@ const docTemplate = `{ "msg_id": { "type": "string" }, + "project_id": { + "type": "string" + }, "request_http_header": { "$ref": "#/definitions/datastore.HttpHeader" }, diff --git a/docs/swagger.json b/docs/swagger.json index 6464c1fc5b..8edd39894e 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5557,6 +5557,9 @@ "msg_id": { "type": "string" }, + "project_id": { + "type": "string" + }, "request_http_header": { "$ref": "#/definitions/datastore.HttpHeader" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 65cf9eb72b..7d69809a59 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -83,6 +83,8 @@ definitions: type: string msg_id: type: string + project_id: + type: string request_http_header: $ref: '#/definitions/datastore.HttpHeader' response_data: diff --git a/docs/v3/openapi3.json b/docs/v3/openapi3.json index 4c36320f73..53f51e1aba 100644 --- a/docs/v3/openapi3.json +++ b/docs/v3/openapi3.json @@ -128,6 +128,9 @@ "msg_id": { "type": "string" }, + "project_id": { + "type": "string" + }, "request_http_header": { "$ref": "#/components/schemas/datastore.HttpHeader" }, diff --git a/docs/v3/openapi3.yaml b/docs/v3/openapi3.yaml index a8615506f9..98a7b85177 100644 --- a/docs/v3/openapi3.yaml +++ b/docs/v3/openapi3.yaml @@ -83,6 +83,8 @@ components: type: string msg_id: type: string + project_id: + type: string request_http_header: $ref: '#/components/schemas/datastore.HttpHeader' response_data: diff --git a/ee/VERSION b/ee/VERSION index 8194d2e551..6b24557cff 100644 --- a/ee/VERSION +++ b/ee/VERSION @@ -1 +1 @@ -v24.6.1 Enterprise Edition +v24.8.1 Enterprise Edition diff --git a/ee/cmd/main.go b/ee/cmd/main.go index 26cf84186a..76251dc2ad 100644 --- a/ee/cmd/main.go +++ b/ee/cmd/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/frain-dev/convoy/cmd/ff" "os" "github.com/frain-dev/convoy/cmd/bootstrap" @@ -45,7 +46,7 @@ func main() { var dbPassword string var dbDatabase string - var fflag string + var fflag []string var redisPort int var redisHost string @@ -91,7 +92,8 @@ func main() { c.Flags().StringVar(&redisDatabase, "redis-database", "", "Redis database") c.Flags().IntVar(&redisPort, "redis-port", 0, "Redis Port") - c.Flags().StringVar(&fflag, "feature-flag", "", "Enable feature flags (experimental)") + c.Flags().StringSliceVar(&fflag, "enable-feature-flag", []string{}, "List of feature flags to enable e.g. \"full-text-search,prometheus\"") + c.Flags().BoolVar(&enableProfiling, "enable-profiling", false, "Enable profiling") // tracing @@ -103,7 +105,7 @@ func main() { c.Flags().StringVar(&otelAuthHeaderValue, "otel-auth-header-value", "", "OTel backend auth header value") // metrics - c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('experimental' feature flag level required") + c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('prometheus' feature flag required") c.Flags().Uint64Var(&prometheusMetricsSampleTime, "metrics-prometheus-sample-time", 5, "Prometheus metrics sample time") c.Flags().Uint64Var(&maxRetrySeconds, "max-retry-seconds", 7200, "Max retry seconds exponential backoff") @@ -122,6 +124,7 @@ func main() { c.AddCommand(ingest.AddIngestCommand(app)) c.AddCommand(stream.AddStreamCommand(app)) c.AddCommand(bootstrap.AddBootstrapCommand(app)) + c.AddCommand(ff.AddFeatureFlagsCommand()) if err := c.Execute(); err != nil { slog.Fatal(err) diff --git a/ee/cmd/server/server.go b/ee/cmd/server/server.go index 2d4a1b5d8d..3078cc19d5 100644 --- a/ee/cmd/server/server.go +++ b/ee/cmd/server/server.go @@ -2,9 +2,10 @@ package server import ( "errors" - "github.com/frain-dev/convoy/internal/pkg/fflag" "time" + "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/worker" @@ -135,7 +136,7 @@ func StartConvoyServer(a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag() + flag, err := fflag.NewFFlag(&cfg) if err != nil { a.Logger.WithError(err).Fatal("failed to create fflag controller") } @@ -157,12 +158,13 @@ func StartConvoyServer(a *cli.App) error { handler, err := api.NewEHandler( &types.APIOptions{ - FFlag: flag, - DB: a.DB, - Queue: a.Queue, - Logger: lo, - Cache: a.Cache, - Rate: a.Rate, + FFlag: flag, + DB: a.DB, + Queue: a.Queue, + Logger: lo, + Cache: a.Cache, + Rate: a.Rate, + Licenser: a.Licenser, }) if err != nil { return err diff --git a/generate.go b/generate.go index 93e3d71b3a..191add2520 100644 --- a/generate.go +++ b/generate.go @@ -10,3 +10,4 @@ package convoy //go:generate mockgen --source internal/pkg/pubsub/pubsub.go --destination mocks/pubsub.go -package mocks //go:generate mockgen --source internal/pkg/dedup/dedup.go --destination mocks/dedup.go -package mocks //go:generate mockgen --source internal/pkg/memorystore/table.go --destination mocks/table.go -package mocks +//go:generate mockgen --source internal/pkg/license/license.go --destination mocks/license.go -package mocks diff --git a/go.mod b/go.mod index da423abb1e..56257ffc23 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/frain-dev/convoy -go 1.21 +go 1.21.0 + +toolchain go1.22.3 require ( cloud.google.com/go/pubsub v1.33.0 @@ -9,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.44.327 github.com/danvixent/asynqmon v0.7.3 github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 - github.com/docker/compose/v2 v2.28.1 + github.com/docker/compose/v2 v2.29.1 github.com/dop251/goja v0.0.0-20231014103939-873a1496dc8e github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562 github.com/exaring/otelpgx v0.5.4 @@ -35,7 +37,8 @@ require ( github.com/jaswdr/faker v1.10.2 github.com/jmoiron/sqlx v1.3.5 github.com/kelseyhightower/envconfig v1.4.0 - github.com/lib/pq v1.10.9 + github.com/keygen-sh/keygen-go/v3 v3.2.0 + github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 github.com/mixpanel/mixpanel-go v1.2.1 github.com/newrelic/go-agent/v3 v3.20.4 @@ -69,7 +72,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.23.0 - google.golang.org/api v0.128.0 + google.golang.org/api v0.149.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/guregu/null.v4 v4.0.0 ) @@ -81,7 +84,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.11.5 // indirect + github.com/Microsoft/hcsshim v0.11.7 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect @@ -99,9 +102,10 @@ require ( github.com/aws/smithy-go v1.19.0 // indirect github.com/buger/goterm v1.0.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/compose-spec/compose-go/v2 v2.1.3 // indirect + github.com/compose-spec/compose-go/v2 v2.1.5 // indirect github.com/containerd/console v1.0.4 // indirect - github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/containerd v1.7.20 // indirect + github.com/containerd/containerd/api v1.7.19 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -111,11 +115,12 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect - github.com/docker/buildx v0.15.1 // indirect - github.com/docker/cli v27.0.3+incompatible // indirect + github.com/docker/buildx v0.16.0 // indirect + github.com/docker/cli v27.1.0+incompatible // indirect + github.com/docker/cli-docs-tool v0.8.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v27.0.3+incompatible // indirect - github.com/docker/docker-credential-helpers v0.8.0 // indirect + github.com/docker/docker v27.1.0+incompatible // indirect + github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect @@ -130,7 +135,7 @@ require ( github.com/go-redis/redis/v7 v7.4.1 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-viper/mapstructure/v2 v2.0.0 // indirect - github.com/gofrs/flock v0.8.1 // indirect + github.com/gofrs/flock v0.12.0 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect @@ -138,8 +143,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect - github.com/google/s2a-go v0.1.5 // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -156,6 +161,8 @@ require ( github.com/jonboulle/clockwork v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/keygen-sh/go-update v1.0.0 // indirect + github.com/keygen-sh/jsonapi-go v1.2.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -164,8 +171,9 @@ require ( github.com/mattn/go-shellwords v1.0.12 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/buildkit v0.14.1 // indirect + github.com/moby/buildkit v0.15.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect @@ -182,6 +190,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -204,6 +213,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -217,7 +227,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect @@ -225,8 +235,8 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect golang.org/x/term v0.20.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/api v0.29.2 // indirect @@ -242,10 +252,9 @@ require ( ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute v1.23.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go v0.110.10 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -259,20 +268,20 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sergi/go-diff v1.0.0 // indirect github.com/spf13/cast v1.5.1 // indirect @@ -280,16 +289,15 @@ require ( github.com/vmihailenco/msgpack/v5 v5.3.5 github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/grpc v1.59.0 + google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect + google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index f2651855ec..398e2ebb03 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= @@ -112,12 +112,10 @@ cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQH cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= @@ -198,8 +196,8 @@ cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= @@ -209,8 +207,8 @@ cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0 cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= -cloud.google.com/go/kms v1.15.3 h1:RYsbxTRmk91ydKCzekI2YjryO4c5Y2M80Zwcs9/D/cI= -cloud.google.com/go/kms v1.15.3/go.mod h1:AJdXqHxS2GlPyduM99s9iGqi2nwbviBbhV/hdmt4iOQ= +cloud.google.com/go/kms v1.15.5 h1:pj1sRfut2eRbD9pFRjNnPNg/CzJPuQAzUujMIM1vVeM= +cloud.google.com/go/kms v1.15.5/go.mod h1:cU2H5jnp6G2TDpUGZyqTCoy1n16fbubHZjmVXSMtwDI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= @@ -435,8 +433,8 @@ github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+V github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim v0.9.1/go.mod h1:Y/0uV2jUab5kBI7SQgl62at0AVX7uaruzADAVmxm3eM= -github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38= -github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -601,8 +599,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= -github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi1KW7R5esrLE= -github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= +github.com/compose-spec/compose-go/v2 v2.1.5 h1:6YoC9ik3NXdSYtgRn51EMZ2DxzGPyGjZ8M0B7mXTXeQ= +github.com/compose-spec/compose-go/v2 v2.1.5/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -643,8 +641,10 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= -github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/containerd v1.7.20 h1:Sl6jQYk3TRavaU83h66QMbI2Nqg9Jm6qzwX57Vsn1SQ= +github.com/containerd/containerd v1.7.20/go.mod h1:52GsS5CwquuqPuLncsXwG0t2CiUce+KsNHJZQJvAgR0= +github.com/containerd/containerd/api v1.7.19 h1:VWbJL+8Ap4Ju2mx9c9qS1uFSB1OVYr5JJrW2yT5vFoA= +github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -758,6 +758,8 @@ github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/deepmap/oapi-codegen v1.9.0/go.mod h1:7t4DbSxmAffcTEgrWvsPYEE2aOARZ8ZKWp3hDuZkHNc= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= @@ -773,13 +775,15 @@ github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnm github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/buildx v0.15.1 h1:1cO6JIc0rOoC8tlxfXoh1HH1uxaNvYH1q7J7kv5enhw= -github.com/docker/buildx v0.15.1/go.mod h1:16DQgJqoggmadc1UhLaUTPqKtR+PlByN/kyXFdkhFCo= +github.com/docker/buildx v0.16.0 h1:LurEflyb6BBoLtDwJY1dw9dLHKzEgGvCjAz67QI0xO0= +github.com/docker/buildx v0.16.0/go.mod h1:4xduW7BOJ2B11AyORKZFDKjF6Vcb4EgTYnV2nunxv9I= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v27.0.3+incompatible h1:usGs0/BoBW8MWxGeEtqPMkzOY56jZ6kYlSN5BLDioCQ= -github.com/docker/cli v27.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/compose/v2 v2.28.1 h1:ORPfiVHrpnRQBDoC3F8JJyWAY8N5gWuo3FgwyivxFdM= -github.com/docker/compose/v2 v2.28.1/go.mod h1:wDtGQFHe99sPLCHXeVbCkc+Wsl4Y/2ZxiAJa/nga6rA= +github.com/docker/cli v27.1.0+incompatible h1:P0KSYmPtNbmx59wHZvG6+rjivhKDRA1BvvWM0f5DgHc= +github.com/docker/cli v27.1.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli-docs-tool v0.8.0 h1:YcDWl7rQJC3lJ7WVZRwSs3bc9nka97QLWfyJQli8yJU= +github.com/docker/cli-docs-tool v0.8.0/go.mod h1:8TQQ3E7mOXoYUs811LiPdUnAhXrcVsBIrW21a5pUbdk= +github.com/docker/compose/v2 v2.29.1 h1:H8oyStDcpjFvyczQ5IuKIE+Bz9jEh8NsRhrAFFlH90U= +github.com/docker/compose/v2 v2.29.1/go.mod h1:oxZ/omiQ4pHlVtp4dJ/B/fZ1pCDlBvYt97UnVelbNKk= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -788,11 +792,11 @@ github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE= -github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= +github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= -github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -1007,8 +1011,8 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.24.2/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/flock v0.12.0 h1:xHW8t8GPAiGtqz7KxiSqfOEXwpOaqhpYZrTE2MQBgXY= +github.com/gofrs/flock v0.12.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -1120,11 +1124,11 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= -github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1137,8 +1141,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -1200,6 +1204,8 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1: github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -1311,6 +1317,12 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/keygen-sh/go-update v1.0.0 h1:M65sTVUHUO07tEK4l1Hq7u5D4kdEqkcgfdzUt3q3S08= +github.com/keygen-sh/go-update v1.0.0/go.mod h1:wn0UWRHLnBP5hwXtj1IdHZqWlHvIadh2Nn+becFf8Ro= +github.com/keygen-sh/jsonapi-go v1.2.1 h1:NTSIAxl2+7S5fPnKgrYwNjQSWbdKRtrFq26SD8AOkiU= +github.com/keygen-sh/jsonapi-go v1.2.1/go.mod h1:8j9vsLiKyJyDqmt8r3tYaYNmXszq2+cFhoO6QdMdAes= +github.com/keygen-sh/keygen-go/v3 v3.2.0 h1:OJqnGtY6z4ZA434kZqfNVHDmSrN5zq4l4XItcB3tECY= +github.com/keygen-sh/keygen-go/v3 v3.2.0/go.mod h1:YoFyryzXEk6XrbT3H8EUUU+JcIJkQu414TA6CvZgS/E= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1321,8 +1333,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1355,9 +1367,8 @@ github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmt github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= @@ -1438,6 +1449,8 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -1450,8 +1463,8 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mixpanel/mixpanel-go v1.2.1 h1:iykbHKomTJjVoWU95Vt1sjZy4HLt8UOYacMEEEMFBok= github.com/mixpanel/mixpanel-go v1.2.1/go.mod h1:mPGaNhBoZMJuLu8k7Y1KhU5n8Vw13rxQZZjHj+b9RLk= -github.com/moby/buildkit v0.14.1 h1:2epLCZTkn4CikdImtsLtIa++7DzCimrrZCT1sway+oI= -github.com/moby/buildkit v0.14.1/go.mod h1:1XssG7cAqv5Bz1xcGMxJL123iCv5TYN4Z/qf647gfuk= +github.com/moby/buildkit v0.15.0 h1:vnZLThPr9JU6SvItctKoa6NfgPZ8oUApg/TCOaa/SVs= +github.com/moby/buildkit v0.15.0/go.mod h1:oN9S+8I7wF26vrqn9NuAF6dFSyGTfXvtiu9o1NlnnH4= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -1521,6 +1534,8 @@ github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94 h1:YXfl+eCNmAQhVbSNQ85bSi1n4qhUBPW8Qq9Rac4pt/s= +github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94/go.mod h1:WUcXjUd98qaCVFb6j8Xc87MsKeMCXDu9Nk8JRJ9SeC8= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -1601,8 +1616,8 @@ github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.m github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= @@ -1702,8 +1717,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/r3labs/diff/v3 v3.0.1 h1:CBKqf3XmNRHXKmdU7mZP1w7TV0pDyVCis1AUHtA4Xtg= github.com/r3labs/diff/v3 v3.0.1/go.mod h1:f1S9bourRbiM66NskseyUdo0fTmEE0qKrikYJX63dgo= @@ -1888,6 +1903,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c h1:+6wg/4ORAbnSoGDzg2Q1i3CeMcT/jjhye/ZfnBHy7/M= github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= @@ -1995,8 +2012,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= @@ -2065,8 +2082,8 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= @@ -2183,7 +2200,6 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -2205,8 +2221,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2233,8 +2249,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2582,8 +2598,8 @@ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= -google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= -google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2592,8 +2608,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -2704,12 +2718,12 @@ google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnp google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= +google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY= +google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2754,8 +2768,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/pkg/cli/cli.go b/internal/pkg/cli/cli.go index 143ea6406b..31e1894c2a 100644 --- a/internal/pkg/cli/cli.go +++ b/internal/pkg/cli/cli.go @@ -2,6 +2,8 @@ package cli import ( "context" + + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/cache" @@ -14,12 +16,13 @@ import ( // App is the core dependency of the entire binary. type App struct { - Version string - DB database.Database - Queue queue.Queuer - Logger log.StdLogger - Cache cache.Cache - Rate limiter.RateLimiter + Version string + DB database.Database + Queue queue.Queuer + Logger log.StdLogger + Cache cache.Cache + Rate limiter.RateLimiter + Licenser license.Licenser // TODO(subomi): Let's make this cleaner. TracerShutdown func(context.Context) error diff --git a/internal/pkg/fflag/fflag.go b/internal/pkg/fflag/fflag.go index 5b2c97ae56..e65a2111fb 100644 --- a/internal/pkg/fflag/fflag.go +++ b/internal/pkg/fflag/fflag.go @@ -1,33 +1,102 @@ package fflag import ( + "errors" + "fmt" "github.com/frain-dev/convoy/config" + "os" + "sort" + "text/tabwriter" ) +var ErrFeatureNotEnabled = errors.New("this feature is not enabled") + type ( FeatureFlagKey string ) const ( - Prometheus FeatureFlagKey = "prometheus" + Prometheus FeatureFlagKey = "prometheus" + FullTextSearch FeatureFlagKey = "full-text-search" +) + +type ( + FeatureFlagState bool +) + +const ( + enabled FeatureFlagState = true + disabled FeatureFlagState = false ) -var features = map[FeatureFlagKey]config.FlagLevel{ - Prometheus: config.ExperimentalFlagLevel, +var DefaultFeaturesState = map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, } -type FFlag struct{} +type FFlag struct { + Features map[FeatureFlagKey]FeatureFlagState +} -func NewFFlag() (*FFlag, error) { - return &FFlag{}, nil +func NewFFlag(c *config.Configuration) (*FFlag, error) { + f := &FFlag{ + Features: clone(DefaultFeaturesState), + } + for _, flag := range c.EnableFeatureFlag { + switch flag { + case string(Prometheus): + f.Features[Prometheus] = enabled + case string(FullTextSearch): + f.Features[FullTextSearch] = enabled + } + } + return f, nil } -func (c *FFlag) CanAccessFeature(key FeatureFlagKey, cfg *config.Configuration) bool { +func clone(src map[FeatureFlagKey]FeatureFlagState) map[FeatureFlagKey]FeatureFlagState { + dst := make(map[FeatureFlagKey]FeatureFlagState) + for k, v := range src { + dst[k] = v + } + return dst +} + +func (c *FFlag) CanAccessFeature(key FeatureFlagKey) bool { // check for this feature in our feature map - flagLevel, ok := features[key] + state, ok := c.Features[key] if !ok { return false } - return flagLevel <= cfg.FeatureFlag // if the feature level is less than or equal to the cfg level, we can access the feature + return bool(state) +} + +func (c *FFlag) ListFeatures() error { + keys := make([]string, 0, len(c.Features)) + + for k := range c.Features { + keys = append(keys, string(k)) + } + sort.Strings(keys) + + w := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', 0) + _, err := fmt.Fprintln(w, "Features\tState") + if err != nil { + return err + } + + for _, k := range keys { + stateBool := c.Features[FeatureFlagKey(k)] + state := "disabled" + if stateBool { + state = "enabled" + } + + _, err := fmt.Fprintf(w, "%s\t%s\n", k, state) + if err != nil { + return err + } + } + + return w.Flush() } diff --git a/internal/pkg/fflag/fflag_test.go b/internal/pkg/fflag/fflag_test.go new file mode 100644 index 0000000000..72fef28c6f --- /dev/null +++ b/internal/pkg/fflag/fflag_test.go @@ -0,0 +1,211 @@ +package fflag + +import ( + "github.com/frain-dev/convoy/config" + "reflect" + "testing" +) + +func TestFFlag_CanAccessFeature(t *testing.T) { + type fields struct { + Features map[FeatureFlagKey]FeatureFlagState + } + type args struct { + key FeatureFlagKey + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "default state - no prometheus", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: Prometheus, + }, + want: false, + }, + { + name: "default state - search available", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: FullTextSearch, + }, + want: true, + }, + { + name: "all enabled state - prometheus available", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: enabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: Prometheus, + }, + want: true, + }, + { + name: "all enabled state - search available", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: enabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: FullTextSearch, + }, + want: true, + }, + { + name: "all disabled state - no prometheus", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: Prometheus, + }, + want: false, + }, + { + name: "all disabled state - no search", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: FullTextSearch, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &FFlag{ + Features: tt.fields.Features, + } + if got := c.CanAccessFeature(tt.args.key); got != tt.want { + t.Errorf("CanAccessFeature() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewFFlag(t *testing.T) { + type args struct { + c *config.Configuration + } + tests := []struct { + name string + args args + want *FFlag + wantErr bool + }{ + { + name: "default state", + args: args{ + &config.Configuration{}, + }, + want: &FFlag{ + Features: DefaultFeaturesState, + }, + wantErr: false, + }, + { + name: "default state - assert all disabled", + args: args{ + &config.Configuration{}, + }, + want: &FFlag{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + wantErr: false, + }, + { + name: "enabled state - prometheus only", + args: args{ + &config.Configuration{ + EnableFeatureFlag: []string{"prometheus"}, + }, + }, + want: &FFlag{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: enabled, + FullTextSearch: disabled, + }, + }, + wantErr: false, + }, + { + name: "all disabled state - by default", + args: args{ + &config.Configuration{}, + }, + want: &FFlag{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewFFlag(tt.args.c) + if (err != nil) != tt.wantErr { + t.Errorf("NewFFlag() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewFFlag() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/pkg/license/keygen/community.go b/internal/pkg/license/keygen/community.go new file mode 100644 index 0000000000..7b8322599a --- /dev/null +++ b/internal/pkg/license/keygen/community.go @@ -0,0 +1,55 @@ +package keygen + +import ( + "context" + + "github.com/keygen-sh/keygen-go/v3" + + "github.com/frain-dev/convoy/datastore" +) + +const ( + projectLimit = 2 + orgLimit = 1 + userLimit = 1 +) + +func communityLicenser(ctx context.Context, orgRepo datastore.OrganisationRepository, userRepo datastore.UserRepository, projectRepo datastore.ProjectRepository) (*Licenser, error) { + m, err := enforceProjectLimit(ctx, projectRepo) + if err != nil { + return nil, err + } + + return &Licenser{ + planType: CommunityPlan, + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: orgLimit}, + CreateUser: {Limit: userLimit}, + CreateProject: {Limit: projectLimit}, + }, + license: &keygen.License{}, + enabledProjects: m, + orgRepo: orgRepo, + userRepo: userRepo, + projectRepo: projectRepo, + }, nil +} + +func enforceProjectLimit(ctx context.Context, projectRepo datastore.ProjectRepository) (map[string]bool, error) { + projects, err := projectRepo.LoadProjects(ctx, &datastore.ProjectFilter{}) + if err != nil { + return nil, err + } + + if len(projects) > projectLimit { + // enabled projects are not within accepted count, allow only the last projects to be active + projects = projects[len(projects)-projectLimit:] + } + + m := map[string]bool{} + for _, p := range projects { + m[p.UID] = true + } + + return m, nil +} diff --git a/internal/pkg/license/keygen/community_test.go b/internal/pkg/license/keygen/community_test.go new file mode 100644 index 0000000000..c5244a8f7d --- /dev/null +++ b/internal/pkg/license/keygen/community_test.go @@ -0,0 +1,89 @@ +package keygen + +import ( + "context" + "testing" + + "github.com/frain-dev/convoy/datastore" + + "github.com/stretchr/testify/require" + + "github.com/frain-dev/convoy/mocks" + "go.uber.org/mock/gomock" +) + +func Test_communityLicenser(t *testing.T) { + testCases := []struct { + name string + featureList map[Feature]*Properties + expectedFeatureList map[Feature]*Properties + expectedEnabledProjects map[string]bool + dbFn func(projectRepo datastore.ProjectRepository) + }{ + { + name: "should_disable_projects", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + dbFn: func(projectRepo datastore.ProjectRepository) { + pr, _ := projectRepo.(*mocks.MockProjectRepository) + pr.EXPECT().LoadProjects(gomock.Any(), gomock.Any()).Times(1).Return([]*datastore.Project{{UID: "01111111"}, {UID: "02222"}, {UID: "033333"}, {UID: "044444"}}, nil) + }, + expectedFeatureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + expectedEnabledProjects: map[string]bool{ + "033333": true, + "044444": true, + }, + }, + { + name: "should_not_disable_projects", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + dbFn: func(projectRepo datastore.ProjectRepository) { + pr, _ := projectRepo.(*mocks.MockProjectRepository) + pr.EXPECT().LoadProjects(gomock.Any(), gomock.Any()).Times(1).Return([]*datastore.Project{{UID: "033333"}, {UID: "044444"}}, nil) + }, + expectedEnabledProjects: map[string]bool{ + "033333": true, + "044444": true, + }, + expectedFeatureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + orgRepo := mocks.NewMockOrganisationRepository(ctrl) + userRepository := mocks.NewMockUserRepository(ctrl) + projectRepo := mocks.NewMockProjectRepository(ctrl) + + if tc.dbFn != nil { + tc.dbFn(projectRepo) + } + + l, err := communityLicenser(context.Background(), orgRepo, userRepository, projectRepo) + require.NoError(t, err) + + require.Equal(t, tc.expectedFeatureList, l.featureList) + require.Equal(t, tc.expectedEnabledProjects, l.enabledProjects) + require.Equal(t, orgRepo, l.orgRepo) + require.Equal(t, userRepository, l.userRepo) + require.Equal(t, projectRepo, l.projectRepo) + }) + } +} diff --git a/internal/pkg/license/keygen/feature.go b/internal/pkg/license/keygen/feature.go new file mode 100644 index 0000000000..678ea5bc1b --- /dev/null +++ b/internal/pkg/license/keygen/feature.go @@ -0,0 +1,46 @@ +package keygen + +type ( + Feature string + PlanType string +) + +const ( + CreateOrg Feature = "CREATE_ORG" + CreateUser Feature = "CREATE_USER" + CreateProject Feature = "CREATE_PROJECT" + UseForwardProxy Feature = "USE_FORWARD_PROXY" + ExportPrometheusMetrics Feature = "EXPORT_PROMETHEUS_METRICS" + AdvancedEndpointMgmt Feature = "ADVANCED_ENDPOINT_MANAGEMENT" + AdvancedWebhookArchiving Feature = "ADVANCED_WEBHOOK_ARCHIVING" + AdvancedMsgBroker Feature = "ADVANCED_MESSAGE_BROKER" + AdvancedSubscriptions Feature = "ADVANCED_SUBSCRIPTIONS" + WebhookTransformations Feature = "WEBHOOK_TRANSFORMATIONS" + HADeployment Feature = "HA_DEPLOYMENT" + WebhookAnalytics Feature = "WEBHOOK_ANALYTICS" + MutualTLS Feature = "MUTUAL_TLS" + AsynqMonitoring Feature = "ASYNQ_MONITORING" + SynchronousWebhooks Feature = "SYNCHRONOUS_WEBHOOKS" + PortalLinks Feature = "PORTAL_LINKS" + ConsumerPoolTuning Feature = "CONSUMER_POOL_TUNING" + AdvancedWebhookFiltering Feature = "ADVANCED_WEBHOOK_FILTERING" +) + +const ( + CommunityPlan PlanType = "community" + BusinessPlan PlanType = "business" + EnterprisePlan PlanType = "enterprise" +) + +// Properties will hold characteristics for features like organisation +// number limit, but it can also be empty, because certain feature don't need them +type Properties struct { + Limit int64 `mapstructure:"limit" json:"-"` + Allowed bool `json:"allowed"` +} + +type LicenseMetadata struct { + UserLimit int64 `mapstructure:"userLimit"` + OrgLimit int64 `mapstructure:"orgLimit"` + ProjectLimit int64 `mapstructure:"projectLimit"` +} diff --git a/internal/pkg/license/keygen/keygen.go b/internal/pkg/license/keygen/keygen.go new file mode 100644 index 0000000000..981583472f --- /dev/null +++ b/internal/pkg/license/keygen/keygen.go @@ -0,0 +1,472 @@ +package keygen + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "sync" + "time" + + "github.com/mitchellh/mapstructure" + + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/pkg/log" + "github.com/frain-dev/convoy/util" + + "github.com/google/uuid" + "github.com/keygen-sh/keygen-go/v3" +) + +type Licenser struct { + licenseKey string + license *keygen.License + planType PlanType + machineFingerprint string + featureList map[Feature]*Properties + + orgRepo datastore.OrganisationRepository + userRepo datastore.UserRepository + projectRepo datastore.ProjectRepository + + // only for community licenser + mu sync.RWMutex + enabledProjects map[string]bool +} + +type Config struct { + LicenseKey string + OrgRepo datastore.OrganisationRepository + ProjectRepo datastore.ProjectRepository + UserRepo datastore.UserRepository +} + +func init() { + keygen.Account = "8200bc0f-f64f-4a38-a9be-d2b16c8f0deb" + keygen.Product = "08d95b4d-4301-42f9-95af-9713e1b41a3a" + keygen.PublicKey = "14549f18dd23e4644aae6b6fd787e4df5f018bce0c7ae2edd29df83309ea76c2" +} + +func NewKeygenLicenser(c *Config) (*Licenser, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + if util.IsStringEmpty(c.LicenseKey) { + // no license key provided, allow access to only community features + return communityLicenser(ctx, c.OrgRepo, c.UserRepo, c.ProjectRepo) + } + + keygen.LicenseKey = c.LicenseKey + fingerprint := uuid.New().String() + + l, err := keygen.Validate(ctx, fingerprint) + if err != nil && !allowKeygenError(err) { + return nil, fmt.Errorf("failed to validate error: %v", err) + } + + err = checkExpiry(l) + if err != nil { + return nil, err + } + + if l.Metadata == nil { + return nil, fmt.Errorf("license has no metadata") + } + + featureList, err := getFeatureList(ctx, l) + if err != nil { + return nil, err + } + + p := l.Metadata["planType"] + if p == nil { + return nil, fmt.Errorf("license plan type unspecified in metadata") + } + + pt, ok := p.(string) + if !ok { + return nil, fmt.Errorf("license plan type is not a string") + } + + return &Licenser{ + machineFingerprint: fingerprint, + licenseKey: c.LicenseKey, + license: l, + orgRepo: c.OrgRepo, + userRepo: c.UserRepo, + projectRepo: c.ProjectRepo, + planType: PlanType(pt), + featureList: featureList, + }, err +} + +func (k *Licenser) ProjectEnabled(projectID string) bool { + k.mu.RLock() + defer k.mu.RUnlock() + if k.enabledProjects == nil { // not community licenser + return true + } + + return k.enabledProjects[projectID] +} + +func (k *Licenser) AddEnabledProject(projectID string) { + k.mu.Lock() + defer k.mu.Unlock() + if k.enabledProjects == nil { // not community licenser + return + } + + if len(k.enabledProjects) == projectLimit { + return + } + + k.enabledProjects[projectID] = true +} + +func (k *Licenser) RemoveEnabledProject(projectID string) { + k.mu.Lock() + defer k.mu.Unlock() + if k.enabledProjects == nil { // not community licenser + return + } + + delete(k.enabledProjects, projectID) +} + +func (k *Licenser) Activate() error { + if util.IsStringEmpty(k.licenseKey) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + machine, err := k.license.Activate(ctx, k.machineFingerprint) + if err != nil { + return fmt.Errorf("failed to activate machine") + } + + // Start a heartbeat monitor for the current machine + err = machine.Monitor(ctx) + if err != nil { + return fmt.Errorf("failed to start machine monitor") + } + + go func() { + // Listen for interrupt and deactivate the machine, if the instance crashes unexpectedly the + // heartbeat monitor helps to tell keygen that this machine should be deactivated + // See the Check-out/check-in licenses section on + // https://keygen.sh/docs/choosing-a-licensing-model/floating-licenses/ + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + <-quit + + if err := machine.Deactivate(ctx); err != nil { + log.WithError(err).Error("failed to deactivate machine") + } + }() + + return nil +} + +func allowKeygenError(err error) bool { + switch { + case errors.Is(err, keygen.ErrLicenseNotActivated): + return true + case errors.Is(err, keygen.ErrLicenseExpired): + return true + case errors.Is(err, keygen.ErrHeartbeatRequired): + return true + } + + return false +} + +var ErrLicenseExpired = errors.New("license expired") + +func checkExpiry(l *keygen.License) error { + if l.Expiry == nil { + return nil + } + + now := time.Now() + + if now.After(*l.Expiry) { + v := now.Sub(*l.Expiry) + + const days = 21 * 24 * time.Hour // 21 days + + if v < days { // expired in less than 21 days, allow instance to boot + daysAgo := int64(v.Hours() / 24) + log.Warnf("license expired %d days ago, access to features will be revoked in %d days", daysAgo, 21-daysAgo) + return nil + } + + return ErrLicenseExpired + } + + return nil +} + +func getFeatureList(ctx context.Context, l *keygen.License) (map[Feature]*Properties, error) { + entitlements, err := l.Entitlements(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load license entitlements: %v", err) + } + + if len(entitlements) == 0 { + return nil, fmt.Errorf("license has no entitlements") + } + + featureList := map[Feature]*Properties{} + for _, entitlement := range entitlements { + featureList[Feature(entitlement.Code)] = &Properties{Allowed: true} + } + + meta := LicenseMetadata{} + if l.Metadata != nil { + err = mapstructure.Decode(l.Metadata, &meta) + if err != nil { + return nil, fmt.Errorf("failed to decode license metadata: %v", err) + } + } + + if meta.OrgLimit != 0 { + featureList[CreateOrg] = &Properties{Limit: meta.OrgLimit} + } + + if meta.UserLimit != 0 { + featureList[CreateUser] = &Properties{Limit: meta.UserLimit} + } + + if meta.ProjectLimit != 0 { + featureList[CreateProject] = &Properties{Limit: meta.ProjectLimit} + } + + return featureList, err +} + +func (k *Licenser) CreateOrg(ctx context.Context) (bool, error) { + err := checkExpiry(k.license) + if err != nil { + return false, err + } + + c, err := k.orgRepo.CountOrganisations(ctx) + if err != nil { + return false, err + } + + p := k.featureList[CreateOrg] + + if p.Limit == -1 { // no limit + return true, nil + } + + if c >= p.Limit { + return false, nil + } + + return true, nil +} + +func (k *Licenser) CreateUser(ctx context.Context) (bool, error) { + err := checkExpiry(k.license) + if err != nil { + return false, err + } + + c, err := k.userRepo.CountUsers(ctx) + if err != nil { + return false, err + } + + p := k.featureList[CreateUser] + + if p.Limit == -1 { // no limit + return true, nil + } + + if c >= p.Limit { + return false, nil + } + + return true, nil +} + +func (k *Licenser) CreateProject(ctx context.Context) (bool, error) { + err := checkExpiry(k.license) + if err != nil { + return false, err + } + + c, err := k.projectRepo.CountProjects(ctx) + if err != nil { + return false, err + } + + p := k.featureList[CreateProject] + + if p.Limit == -1 { // no limit + return true, nil + } + + if c >= p.Limit { + return false, nil + } + + return true, nil +} + +func (k *Licenser) UseForwardProxy() bool { + if checkExpiry(k.license) != nil { + return false + } + + _, ok := k.featureList[UseForwardProxy] + return ok +} + +func (k *Licenser) CanExportPrometheusMetrics() bool { + if checkExpiry(k.license) != nil { + return false + } + + _, ok := k.featureList[ExportPrometheusMetrics] + return ok +} + +func (k *Licenser) AdvancedEndpointMgmt() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedEndpointMgmt] + return ok +} + +func (k *Licenser) AdvancedRetentionPolicy() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedWebhookArchiving] + return ok +} + +func (k *Licenser) AdvancedMsgBroker() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedMsgBroker] + return ok +} + +func (k *Licenser) AdvancedSubscriptions() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedSubscriptions] + return ok +} + +func (k *Licenser) Transformations() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[WebhookTransformations] + return ok +} + +func (k *Licenser) HADeployment() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[HADeployment] + return ok +} + +func (k *Licenser) WebhookAnalytics() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[WebhookAnalytics] + return ok +} + +func (k *Licenser) MutualTLS() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[MutualTLS] + return ok +} + +func (k *Licenser) AsynqMonitoring() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AsynqMonitoring] + return ok +} + +func (k *Licenser) SynchronousWebhooks() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[SynchronousWebhooks] + return ok +} + +func (k *Licenser) PortalLinks() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[PortalLinks] + return ok +} + +func (k *Licenser) ConsumerPoolTuning() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[ConsumerPoolTuning] + return ok +} + +func (k *Licenser) AdvancedWebhookFiltering() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedWebhookFiltering] + return ok +} + +func (k *Licenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { + // only these guys have dynamic limits for now + for f := range k.featureList { + switch f { + case CreateOrg: + ok, err := k.CreateOrg(ctx) + if err != nil { + return nil, err + } + k.featureList[f].Allowed = ok + case CreateUser: + ok, err := k.CreateUser(ctx) + if err != nil { + return nil, err + } + k.featureList[f].Allowed = ok + case CreateProject: + ok, err := k.CreateProject(ctx) + if err != nil { + return nil, err + } + k.featureList[f].Allowed = ok + } + } + + return json.Marshal(k.featureList) +} diff --git a/internal/pkg/license/keygen/keygen_test.go b/internal/pkg/license/keygen/keygen_test.go new file mode 100644 index 0000000000..a14738f807 --- /dev/null +++ b/internal/pkg/license/keygen/keygen_test.go @@ -0,0 +1,668 @@ +package keygen + +import ( + "context" + "encoding/json" + "errors" + "math" + "testing" + "time" + + "github.com/frain-dev/convoy/mocks" + "github.com/keygen-sh/keygen-go/v3" + "go.uber.org/mock/gomock" + + "github.com/stretchr/testify/require" +) + +func TestKeygenLicenserBoolMethods(t *testing.T) { + k := Licenser{featureList: map[Feature]*Properties{UseForwardProxy: {}}, license: &keygen.License{}} + require.True(t, k.UseForwardProxy()) + + k = Licenser{featureList: map[Feature]*Properties{UseForwardProxy: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.UseForwardProxy()) + + k = Licenser{featureList: map[Feature]*Properties{ExportPrometheusMetrics: {}}, license: &keygen.License{}} + require.True(t, k.CanExportPrometheusMetrics()) + + k = Licenser{featureList: map[Feature]*Properties{ExportPrometheusMetrics: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.CanExportPrometheusMetrics()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedEndpointMgmt: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedEndpointMgmt()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedEndpointMgmt: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedEndpointMgmt()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookArchiving: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedRetentionPolicy()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookArchiving: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedRetentionPolicy()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedMsgBroker: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedMsgBroker()) + k = Licenser{featureList: map[Feature]*Properties{AdvancedMsgBroker: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedMsgBroker()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedSubscriptions: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedSubscriptions()) + k = Licenser{featureList: map[Feature]*Properties{AdvancedSubscriptions: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedSubscriptions()) + + k = Licenser{featureList: map[Feature]*Properties{WebhookTransformations: {}}, license: &keygen.License{}} + require.True(t, k.Transformations()) + k = Licenser{featureList: map[Feature]*Properties{WebhookTransformations: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.Transformations()) + + k = Licenser{featureList: map[Feature]*Properties{HADeployment: {}}, license: &keygen.License{}} + require.True(t, k.HADeployment()) + k = Licenser{featureList: map[Feature]*Properties{HADeployment: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.HADeployment()) + + k = Licenser{featureList: map[Feature]*Properties{WebhookAnalytics: {}}, license: &keygen.License{}} + require.True(t, k.WebhookAnalytics()) + k = Licenser{featureList: map[Feature]*Properties{WebhookAnalytics: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.WebhookAnalytics()) + + k = Licenser{featureList: map[Feature]*Properties{MutualTLS: {}}, license: &keygen.License{}} + require.True(t, k.MutualTLS()) + k = Licenser{featureList: map[Feature]*Properties{MutualTLS: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.MutualTLS()) + + k = Licenser{featureList: map[Feature]*Properties{AsynqMonitoring: {}}, license: &keygen.License{}} + require.True(t, k.AsynqMonitoring()) + + k = Licenser{featureList: map[Feature]*Properties{AsynqMonitoring: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AsynqMonitoring()) + + k = Licenser{featureList: map[Feature]*Properties{SynchronousWebhooks: {}}, license: &keygen.License{}} + require.True(t, k.SynchronousWebhooks()) + k = Licenser{featureList: map[Feature]*Properties{SynchronousWebhooks: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.SynchronousWebhooks()) + + k = Licenser{featureList: map[Feature]*Properties{PortalLinks: {}}, license: &keygen.License{}} + require.True(t, k.PortalLinks()) + k = Licenser{featureList: map[Feature]*Properties{PortalLinks: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.PortalLinks()) + + k = Licenser{featureList: map[Feature]*Properties{ConsumerPoolTuning: {}}, license: &keygen.License{}} + require.True(t, k.ConsumerPoolTuning()) + k = Licenser{featureList: map[Feature]*Properties{ConsumerPoolTuning: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.ConsumerPoolTuning()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookFiltering: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedWebhookFiltering()) + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookFiltering: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedWebhookFiltering()) + + k = Licenser{enabledProjects: map[string]bool{ + "12345": true, + }} + require.True(t, k.ProjectEnabled("12345")) + require.False(t, k.ProjectEnabled("5555")) + + // when k.enabledProjects is nil, should not add anything + k = Licenser{} + k.AddEnabledProject("11111") + require.False(t, k.enabledProjects["11111"]) + + k = Licenser{enabledProjects: map[string]bool{}} + k.AddEnabledProject("11111") + require.True(t, k.enabledProjects["11111"]) + + k = Licenser{enabledProjects: map[string]bool{"11111": true, "2222": true}} + k.RemoveEnabledProject("11111") + require.NotContains(t, k.enabledProjects, "11111") + require.Contains(t, k.enabledProjects, "2222") + + falseLicenser := Licenser{featureList: map[Feature]*Properties{}, license: &keygen.License{Expiry: timePtr(time.Now().Add(400000 * time.Hour))}} + + require.False(t, falseLicenser.UseForwardProxy()) + require.False(t, falseLicenser.PortalLinks()) + require.False(t, falseLicenser.CanExportPrometheusMetrics()) + require.False(t, falseLicenser.AdvancedEndpointMgmt()) + require.False(t, falseLicenser.AdvancedRetentionPolicy()) + require.False(t, falseLicenser.AdvancedMsgBroker()) + require.False(t, falseLicenser.AdvancedSubscriptions()) + require.False(t, falseLicenser.Transformations()) + require.False(t, falseLicenser.HADeployment()) + require.False(t, falseLicenser.WebhookAnalytics()) + require.False(t, falseLicenser.MutualTLS()) + require.False(t, falseLicenser.AsynqMonitoring()) + require.False(t, falseLicenser.SynchronousWebhooks()) +} + +func provideLicenser(ctrl *gomock.Controller, license *keygen.License, fl map[Feature]*Properties) *Licenser { + return &Licenser{ + featureList: fl, + license: license, + orgRepo: mocks.NewMockOrganisationRepository(ctrl), + userRepo: mocks.NewMockUserRepository(ctrl), + projectRepo: mocks.NewMockProjectRepository(ctrl), + } +} + +func TestKeygenLicenser_CreateProject(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + ctx context.Context + dbFn func(k *Licenser) + license *keygen.License + want bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_return_true", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_return_false_for_license_expired", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(-time.Hour * 40000))}, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + { + name: "should_return_false_for_limit_reached", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(1), nil) + }, + ctx: context.Background(), + want: false, + wantErr: false, + }, + { + name: "should_return_true_for_no_limit", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: -1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(math.MaxInt64), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_error_for_failed_to_count_org", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), errors.New("failed")) + }, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: "failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, tt.license, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.CreateProject(tt.ctx) + require.Equal(t, tt.want, got) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + }) + } +} + +func TestKeygenLicenser_CanCreateOrg(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + ctx context.Context + dbFn func(k *Licenser) + license *keygen.License + want bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_return_true", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_return_false_for_license_expired", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * -40000))}, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + { + name: "should_return_false_for_limit_reached", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(1), nil) + }, + ctx: context.Background(), + want: false, + wantErr: false, + }, + { + name: "should_return_true_for_no_limit", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: -1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(math.MaxInt64), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_error_for_failed_to_count_org", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), errors.New("failed")) + }, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: "failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, tt.license, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.CreateOrg(tt.ctx) + require.Equal(t, tt.want, got) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + }) + } +} + +func TestKeygenLicenser_CanCreateUser(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + ctx context.Context + license *keygen.License + dbFn func(k *Licenser) + canCreateMember bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_return_true", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + canCreateMember: true, + wantErr: false, + }, + { + name: "should_return_false_for_limit_reached", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(1), nil) + }, + ctx: context.Background(), + canCreateMember: false, + wantErr: false, + }, + { + name: "should_return_true_for_no_limit", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: -1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + canCreateMember: true, + wantErr: false, + }, + { + name: "should_error_for_failed_to_count_org_members", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(0), errors.New("failed")) + }, + ctx: context.Background(), + canCreateMember: false, + wantErr: true, + wantErrMsg: "failed", + }, + { + name: "should_error_for_license_expired", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * -40000))}, + ctx: context.Background(), + canCreateMember: false, + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, tt.license, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.CreateUser(tt.ctx) + require.Equal(t, tt.canCreateMember, got) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + }) + } +} + +func TestLicenser_FeatureListJSON(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + dbFn func(k *Licenser) + want json.RawMessage + wantErr bool + wantErrMsg string + }{ + { + name: "should_get_feature_list", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":true},"CREATE_PROJECT":{"allowed":true},"CREATE_USER":{"allowed":true}}`), + wantErr: false, + wantErrMsg: "", + }, + + { + name: "should_be_false_create_org", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(1), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":false},"CREATE_PROJECT":{"allowed":true},"CREATE_USER":{"allowed":true}}`), + wantErr: false, + wantErrMsg: "", + }, + + { + name: "should_be_false_create_user", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(1), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":true},"CREATE_PROJECT":{"allowed":true},"CREATE_USER":{"allowed":false}}`), + wantErr: false, + wantErrMsg: "", + }, + + { + name: "should_be_false_create_project", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(2), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":true},"CREATE_PROJECT":{"allowed":false},"CREATE_USER":{"allowed":true}}`), + wantErr: false, + wantErrMsg: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour))}, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.FeatureListJSON(context.Background()) + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestCheckExpiry(t *testing.T) { + tests := []struct { + name string + expiry *time.Time + wantErr bool + wantErrMsg string + }{ + { + name: "No Expiry Date", + expiry: nil, + wantErr: false, + wantErrMsg: "", + }, + { + name: "License Expired within 21 Days", + expiry: timePtr(time.Now().Add(-10 * 24 * time.Hour)), // 10 days ago + wantErr: false, + }, + { + name: "License Expired beyond 21 Days", + expiry: timePtr(time.Now().Add(-22 * 24 * time.Hour)), // 22 days ago + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + { + name: "License Not Yet Expired", + expiry: timePtr(time.Now().Add(5 * 24 * time.Hour)), // 5 days in the future + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + license := &keygen.License{ + Expiry: tt.expiry, + } + + err := checkExpiry(license) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + require.NoError(t, err) + }) + } +} + +// timePtr is a helper function to get a pointer to a time.Time value +func timePtr(t time.Time) *time.Time { + return &t +} diff --git a/internal/pkg/license/license.go b/internal/pkg/license/license.go new file mode 100644 index 0000000000..faf6a693a1 --- /dev/null +++ b/internal/pkg/license/license.go @@ -0,0 +1,47 @@ +package license + +import ( + "context" + "encoding/json" + + "github.com/frain-dev/convoy/internal/pkg/license/keygen" +) + +// Licenser interface provides methods to determine whether the specified license can utilise certain features in convoy. +type Licenser interface { + CreateOrg(ctx context.Context) (bool, error) + CreateUser(ctx context.Context) (bool, error) + CreateProject(ctx context.Context) (bool, error) + UseForwardProxy() bool + CanExportPrometheusMetrics() bool + AdvancedEndpointMgmt() bool + AdvancedSubscriptions() bool + Transformations() bool + AsynqMonitoring() bool + PortalLinks() bool + ConsumerPoolTuning() bool + AdvancedWebhookFiltering() bool + + // need more fleshing out + AdvancedRetentionPolicy() bool + AdvancedMsgBroker() bool + WebhookAnalytics() bool + HADeployment() bool + MutualTLS() bool + SynchronousWebhooks() bool + FeatureListJSON(ctx context.Context) (json.RawMessage, error) + + RemoveEnabledProject(projectID string) + AddEnabledProject(projectID string) + ProjectEnabled(projectID string) bool +} + +var _ Licenser = &keygen.Licenser{} + +type Config struct { + KeyGen keygen.Config +} + +func NewLicenser(c *Config) (Licenser, error) { + return keygen.NewKeygenLicenser(&c.KeyGen) +} diff --git a/internal/pkg/license/noop/noop.go b/internal/pkg/license/noop/noop.go new file mode 100644 index 0000000000..6cc613e271 --- /dev/null +++ b/internal/pkg/license/noop/noop.go @@ -0,0 +1,100 @@ +//go:build integration + +package noop + +import ( + "context" + "encoding/json" +) + +// Noop License is for testing only + +type Licenser struct{} + +func (Licenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { + return []byte{}, nil +} + +func NewLicenser() *Licenser { + return &Licenser{} +} + +func (Licenser) CreateOrg(ctx context.Context) (bool, error) { + return true, nil +} + +func (Licenser) CreateUser(ctx context.Context) (bool, error) { + return true, nil +} + +func (Licenser) CreateProject(ctx context.Context) (bool, error) { + return true, nil +} + +func (Licenser) UseForwardProxy() bool { + return true +} + +func (Licenser) CanExportPrometheusMetrics() bool { + return true +} + +func (Licenser) AdvancedEndpointMgmt() bool { + return true +} + +func (Licenser) AdvancedSubscriptions() bool { + return true +} + +func (Licenser) Transformations() bool { + return true +} + +func (Licenser) AsynqMonitoring() bool { + return true +} + +func (Licenser) AdvancedRetentionPolicy() bool { + return true +} + +func (Licenser) AdvancedMsgBroker() bool { + return true +} + +func (Licenser) WebhookAnalytics() bool { + return true +} + +func (Licenser) HADeployment() bool { + return true +} + +func (Licenser) MutualTLS() bool { + return true +} + +func (Licenser) SynchronousWebhooks() bool { + return true +} + +func (Licenser) RemoveEnabledProject(_ string) {} + +func (Licenser) ProjectEnabled(_ string) bool { + return true +} + +func (Licenser) AddEnabledProject(_ string) {} + +func (Licenser) ConsumerPoolTuning() bool { + return true +} + +func (Licenser) AdvancedWebhookFiltering() bool { + return true +} + +func (Licenser) PortalLinks() bool { + return true +} diff --git a/internal/pkg/metrics/data_plane.go b/internal/pkg/metrics/data_plane.go index 60262c12fb..425ea0c08b 100644 --- a/internal/pkg/metrics/data_plane.go +++ b/internal/pkg/metrics/data_plane.go @@ -1,14 +1,18 @@ package metrics import ( + "sync" + "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/prometheus/client_golang/prometheus" - "sync" ) -var m *Metrics -var once sync.Once +var ( + m *Metrics + once sync.Once +) const ( projectLabel = "project" @@ -24,15 +28,15 @@ type Metrics struct { EventDeliveryLatency *prometheus.HistogramVec } -func GetDPInstance() *Metrics { +func GetDPInstance(licenser license.Licenser) *Metrics { once.Do(func() { - m = newMetrics(Reg()) + m = newMetrics(Reg(), licenser) }) return m } -func newMetrics(pr prometheus.Registerer) *Metrics { - m := InitMetrics() +func newMetrics(pr prometheus.Registerer, licenser license.Licenser) *Metrics { + m := InitMetrics(licenser) if m.IsEnabled && m.IngestTotal != nil && m.IngestConsumedTotal != nil && m.IngestErrorsTotal != nil { pr.MustRegister( @@ -45,15 +49,14 @@ func newMetrics(pr prometheus.Registerer) *Metrics { return m } -func InitMetrics() *Metrics { - +func InitMetrics(licenser license.Licenser) *Metrics { cfg, err := config.Get() if err != nil { return &Metrics{ IsEnabled: false, } } - if !cfg.Metrics.IsEnabled { + if !cfg.Metrics.IsEnabled || !licenser.CanExportPrometheusMetrics() { return &Metrics{ IsEnabled: false, } diff --git a/internal/pkg/middleware/middleware.go b/internal/pkg/middleware/middleware.go index e1c16ce071..b2dd322a67 100644 --- a/internal/pkg/middleware/middleware.go +++ b/internal/pkg/middleware/middleware.go @@ -6,14 +6,15 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/frain-dev/convoy" - "github.com/frain-dev/convoy/internal/pkg/limiter" - rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" "net/http" "strconv" "strings" "time" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/internal/pkg/limiter" + rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" + "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/riandyrn/otelchi" @@ -93,14 +94,7 @@ func WriteRequestIDHeader(next http.Handler) http.Handler { func CanAccessFeature(fflag *fflag.FFlag, featureKey fflag.FeatureFlagKey) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cfg, err := config.Get() - if err != nil { - log.FromContext(r.Context()).WithError(err).Error("failed to load configuration") - _ = render.Render(w, r, util.NewErrorResponse("something went wrong", http.StatusInternalServerError)) - return - } - - if !fflag.CanAccessFeature(featureKey, &cfg) { + if !fflag.CanAccessFeature(featureKey) { _ = render.Render(w, r, util.NewErrorResponse("this feature is not enabled in this server", http.StatusForbidden)) return } diff --git a/internal/pkg/pubsub/amqp/client.go b/internal/pkg/pubsub/amqp/client.go index 40e40f5ac2..a7268decd1 100644 --- a/internal/pkg/pubsub/amqp/client.go +++ b/internal/pkg/pubsub/amqp/client.go @@ -3,7 +3,9 @@ package rqm import ( "context" "fmt" + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/log" @@ -23,10 +25,10 @@ type Amqp struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter) *Amqp { - +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser) *Amqp { return &Amqp{ Cfg: source.PubSub.Amqp, source: source, @@ -34,6 +36,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, } } @@ -79,7 +82,6 @@ func (k *Amqp) Verify() error { defer ch.Close() return nil - } func (k *Amqp) consume() { @@ -135,13 +137,12 @@ func (k *Amqp) consume() { false, // no-wait nil, // args ) - if err != nil { log.WithError(err).Error("failed to consume messages") return } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(k.licenser) mm.IncrementIngestTotal(k.source) for d := range messages { @@ -159,7 +160,6 @@ func (k *Amqp) consume() { mm.IncrementIngestConsumedTotal(k.source) } } else { - // Reject the message and send it to DLQ if err := d.Nack(false, false); err != nil { k.log.WithError(err).Error("failed to nack message") @@ -167,5 +167,4 @@ func (k *Amqp) consume() { } } } - } diff --git a/internal/pkg/pubsub/google/client.go b/internal/pkg/pubsub/google/client.go index 01ba5b5953..fffa79ef0b 100644 --- a/internal/pkg/pubsub/google/client.go +++ b/internal/pkg/pubsub/google/client.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/msgpack" @@ -24,9 +26,10 @@ type Google struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter) *Google { +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser) *Google { return &Google{ Cfg: source.PubSub.Google, source: source, @@ -34,6 +37,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, } } @@ -71,7 +75,6 @@ func (g *Google) Verify() error { func (g *Google) consume() { client, err := pubsub.NewClient(g.ctx, g.Cfg.ProjectID, option.WithCredentialsJSON(g.Cfg.ServiceAccount)) - if err != nil { g.log.WithError(err).Error("failed to create new pubsub client") } @@ -102,7 +105,7 @@ func (g *Google) consume() { attributes = emptyBytes } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(g.licenser) mm.IncrementIngestTotal(g.source) if err := g.handler(ctx, g.source, string(m.Data), attributes); err != nil { diff --git a/internal/pkg/pubsub/ingest.go b/internal/pkg/pubsub/ingest.go index 58b7caeea0..c0752325dc 100644 --- a/internal/pkg/pubsub/ingest.go +++ b/internal/pkg/pubsub/ingest.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/transform" @@ -39,9 +41,10 @@ type Ingest struct { table *memorystore.Table log log.StdLogger instanceId string + licenser license.Licenser } -func NewIngest(ctx context.Context, table *memorystore.Table, queue queue.Queuer, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) (*Ingest, error) { +func NewIngest(ctx context.Context, table *memorystore.Table, queue queue.Queuer, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) (*Ingest, error) { ctx = context.WithValue(ctx, ingestCtx, nil) i := &Ingest{ ctx: ctx, @@ -50,6 +53,7 @@ func NewIngest(ctx context.Context, table *memorystore.Table, queue queue.Queuer queue: queue, rateLimiter: rateLimiter, instanceId: instanceId, + licenser: licenser, sources: make(map[memorystore.Key]*PubSubSource), ticker: time.NewTicker(time.Duration(1) * time.Second), } @@ -118,12 +122,12 @@ func (i *Ingest) run() error { return errors.New("invalid source in memory store") } - ps, err := NewPubSubSource(i.ctx, &ss, i.handler, i.log, i.rateLimiter, i.instanceId) + ps, err := NewPubSubSource(i.ctx, &ss, i.handler, i.log, i.rateLimiter, i.licenser, i.instanceId) if err != nil { return err } - //ps.hash = key + // ps.hash = key ps.Start() i.sources[key] = ps } diff --git a/internal/pkg/pubsub/kafka/client.go b/internal/pkg/pubsub/kafka/client.go index 9660382905..2db2c8c16e 100644 --- a/internal/pkg/pubsub/kafka/client.go +++ b/internal/pkg/pubsub/kafka/client.go @@ -4,11 +4,13 @@ import ( "context" "crypto/tls" "fmt" + "time" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/msgpack" - "time" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/log" @@ -27,10 +29,11 @@ type Kafka struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser instanceId string } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) *Kafka { +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) *Kafka { return &Kafka{ Cfg: source.PubSub.Kafka, source: source, @@ -38,6 +41,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, instanceId: instanceId, } } @@ -107,7 +111,6 @@ func (k *Kafka) Verify() error { } return nil - } func (k *Kafka) consume() { @@ -160,7 +163,7 @@ func (k *Kafka) consume() { continue } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(k.licenser) mm.IncrementIngestTotal(k.source) var d D = m.Headers diff --git a/internal/pkg/pubsub/pubsub.go b/internal/pkg/pubsub/pubsub.go index 9ef88d4ea5..a4e4b35b10 100644 --- a/internal/pkg/pubsub/pubsub.go +++ b/internal/pkg/pubsub/pubsub.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "fmt" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -33,20 +35,22 @@ type PubSubSource struct { // The DB source source *datastore.Source + licenser license.Licenser + // This is a hash for the source config used to // track if an existing source config has been changed. hash string } -func NewPubSubSource(ctx context.Context, source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) (*PubSubSource, error) { - client, err := createClient(source, handler, log, rateLimiter, instanceId) +func NewPubSubSource(ctx context.Context, source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) (*PubSubSource, error) { + client, err := createClient(source, handler, log, rateLimiter, licenser, instanceId) if err != nil { return nil, err } ctx, cancelFunc := context.WithCancel(ctx) - pubSubSource := &PubSubSource{ctx: ctx, cancelFunc: cancelFunc, client: client, source: source} - //pubSubSource.hash = generateSourceKey(source) + pubSubSource := &PubSubSource{ctx: ctx, cancelFunc: cancelFunc, client: client, licenser: licenser, source: source} + // pubSubSource.hash = generateSourceKey(source) pubSubSource.cancelFunc = cancelFunc return pubSubSource, nil @@ -60,21 +64,21 @@ func (p *PubSubSource) Stop() { p.cancelFunc() } -func createClient(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) (PubSub, error) { +func createClient(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) (PubSub, error) { if source.PubSub.Type == datastore.SqsPubSub { - return sqs.New(source, handler, log, rateLimiter, instanceId), nil + return sqs.New(source, handler, log, rateLimiter, licenser, instanceId), nil } if source.PubSub.Type == datastore.GooglePubSub { - return google.New(source, handler, log, rateLimiter), nil + return google.New(source, handler, log, rateLimiter, licenser), nil } if source.PubSub.Type == datastore.KafkaPubSub { - return kafka.New(source, handler, log, rateLimiter, instanceId), nil + return kafka.New(source, handler, log, rateLimiter, licenser, instanceId), nil } if source.PubSub.Type == datastore.AmqpPubSub { - return rqm.New(source, handler, log, rateLimiter), nil + return rqm.New(source, handler, log, rateLimiter, licenser), nil } return nil, fmt.Errorf("pub sub type %s is not supported", source.PubSub.Type) diff --git a/internal/pkg/pubsub/sqs/client.go b/internal/pkg/pubsub/sqs/client.go index 624f7472a7..61740d79a6 100644 --- a/internal/pkg/pubsub/sqs/client.go +++ b/internal/pkg/pubsub/sqs/client.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" + "sync" + "time" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/msgpack" - "sync" - "time" "github.com/frain-dev/convoy/util" @@ -31,10 +33,11 @@ type Sqs struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser instanceId string } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) *Sqs { +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) *Sqs { return &Sqs{ Cfg: source.PubSub.Sqs, source: source, @@ -42,6 +45,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, instanceId: instanceId, } } @@ -60,7 +64,6 @@ func (s *Sqs) Verify() error { Region: aws.String(s.Cfg.DefaultRegion), Credentials: credentials.NewStaticCredentials(s.Cfg.AccessKeyID, s.Cfg.SecretKey, ""), }) - if err != nil { log.WithError(err).Error("failed to create new session - sqs") return ErrInvalidCredentials @@ -95,7 +98,6 @@ func (s *Sqs) consume() { url, err := svc.GetQueueUrl(&sqs.GetQueueUrlInput{ QueueName: &s.Cfg.QueueName, }) - if err != nil { s.log.WithError(err).Error("failed to fetch queue url - sqs") } @@ -146,13 +148,12 @@ func (s *Sqs) consume() { WaitTimeSeconds: aws.Int64(1), MessageAttributeNames: []*string{&allAttr}, }) - if err != nil { s.log.WithError(err).Error("failed to fetch message - sqs") continue } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(s.licenser) mm.IncrementIngestTotal(s.source) var wg sync.WaitGroup @@ -197,7 +198,6 @@ func (s *Sqs) consume() { s.log.WithError(err).Error("failed to delete message") } } - }(message) wg.Wait() diff --git a/mocks/license.go b/mocks/license.go new file mode 100644 index 0000000000..29d8166aaa --- /dev/null +++ b/mocks/license.go @@ -0,0 +1,349 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/license/license.go +// +// Generated by this command: +// +// mockgen --source internal/pkg/license/license.go --destination mocks/license.go -package mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + json "encoding/json" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockLicenser is a mock of Licenser interface. +type MockLicenser struct { + ctrl *gomock.Controller + recorder *MockLicenserMockRecorder +} + +// MockLicenserMockRecorder is the mock recorder for MockLicenser. +type MockLicenserMockRecorder struct { + mock *MockLicenser +} + +// NewMockLicenser creates a new mock instance. +func NewMockLicenser(ctrl *gomock.Controller) *MockLicenser { + mock := &MockLicenser{ctrl: ctrl} + mock.recorder = &MockLicenserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLicenser) EXPECT() *MockLicenserMockRecorder { + return m.recorder +} + +// AddEnabledProject mocks base method. +func (m *MockLicenser) AddEnabledProject(projectID string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddEnabledProject", projectID) +} + +// AddEnabledProject indicates an expected call of AddEnabledProject. +func (mr *MockLicenserMockRecorder) AddEnabledProject(projectID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddEnabledProject", reflect.TypeOf((*MockLicenser)(nil).AddEnabledProject), projectID) +} + +// AdvancedEndpointMgmt mocks base method. +func (m *MockLicenser) AdvancedEndpointMgmt() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedEndpointMgmt") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedEndpointMgmt indicates an expected call of AdvancedEndpointMgmt. +func (mr *MockLicenserMockRecorder) AdvancedEndpointMgmt() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedEndpointMgmt", reflect.TypeOf((*MockLicenser)(nil).AdvancedEndpointMgmt)) +} + +// AdvancedMsgBroker mocks base method. +func (m *MockLicenser) AdvancedMsgBroker() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedMsgBroker") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedMsgBroker indicates an expected call of AdvancedMsgBroker. +func (mr *MockLicenserMockRecorder) AdvancedMsgBroker() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedMsgBroker", reflect.TypeOf((*MockLicenser)(nil).AdvancedMsgBroker)) +} + +// AdvancedRetentionPolicy mocks base method. +func (m *MockLicenser) AdvancedRetentionPolicy() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedRetentionPolicy") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedRetentionPolicy indicates an expected call of AdvancedRetentionPolicy. +func (mr *MockLicenserMockRecorder) AdvancedRetentionPolicy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedRetentionPolicy", reflect.TypeOf((*MockLicenser)(nil).AdvancedRetentionPolicy)) +} + +// AdvancedSubscriptions mocks base method. +func (m *MockLicenser) AdvancedSubscriptions() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedSubscriptions") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedSubscriptions indicates an expected call of AdvancedSubscriptions. +func (mr *MockLicenserMockRecorder) AdvancedSubscriptions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedSubscriptions", reflect.TypeOf((*MockLicenser)(nil).AdvancedSubscriptions)) +} + +// AdvancedWebhookFiltering mocks base method. +func (m *MockLicenser) AdvancedWebhookFiltering() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedWebhookFiltering") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedWebhookFiltering indicates an expected call of AdvancedWebhookFiltering. +func (mr *MockLicenserMockRecorder) AdvancedWebhookFiltering() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedWebhookFiltering", reflect.TypeOf((*MockLicenser)(nil).AdvancedWebhookFiltering)) +} + +// AsynqMonitoring mocks base method. +func (m *MockLicenser) AsynqMonitoring() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AsynqMonitoring") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AsynqMonitoring indicates an expected call of AsynqMonitoring. +func (mr *MockLicenserMockRecorder) AsynqMonitoring() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsynqMonitoring", reflect.TypeOf((*MockLicenser)(nil).AsynqMonitoring)) +} + +// CanExportPrometheusMetrics mocks base method. +func (m *MockLicenser) CanExportPrometheusMetrics() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CanExportPrometheusMetrics") + ret0, _ := ret[0].(bool) + return ret0 +} + +// CanExportPrometheusMetrics indicates an expected call of CanExportPrometheusMetrics. +func (mr *MockLicenserMockRecorder) CanExportPrometheusMetrics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanExportPrometheusMetrics", reflect.TypeOf((*MockLicenser)(nil).CanExportPrometheusMetrics)) +} + +// ConsumerPoolTuning mocks base method. +func (m *MockLicenser) ConsumerPoolTuning() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConsumerPoolTuning") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ConsumerPoolTuning indicates an expected call of ConsumerPoolTuning. +func (mr *MockLicenserMockRecorder) ConsumerPoolTuning() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsumerPoolTuning", reflect.TypeOf((*MockLicenser)(nil).ConsumerPoolTuning)) +} + +// CreateOrg mocks base method. +func (m *MockLicenser) CreateOrg(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrg", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateOrg indicates an expected call of CreateOrg. +func (mr *MockLicenserMockRecorder) CreateOrg(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrg", reflect.TypeOf((*MockLicenser)(nil).CreateOrg), ctx) +} + +// CreateProject mocks base method. +func (m *MockLicenser) CreateProject(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateProject", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateProject indicates an expected call of CreateProject. +func (mr *MockLicenserMockRecorder) CreateProject(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProject", reflect.TypeOf((*MockLicenser)(nil).CreateProject), ctx) +} + +// CreateUser mocks base method. +func (m *MockLicenser) CreateUser(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateUser", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateUser indicates an expected call of CreateUser. +func (mr *MockLicenserMockRecorder) CreateUser(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateUser", reflect.TypeOf((*MockLicenser)(nil).CreateUser), ctx) +} + +// FeatureListJSON mocks base method. +func (m *MockLicenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FeatureListJSON", ctx) + ret0, _ := ret[0].(json.RawMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FeatureListJSON indicates an expected call of FeatureListJSON. +func (mr *MockLicenserMockRecorder) FeatureListJSON(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FeatureListJSON", reflect.TypeOf((*MockLicenser)(nil).FeatureListJSON), ctx) +} + +// HADeployment mocks base method. +func (m *MockLicenser) HADeployment() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HADeployment") + ret0, _ := ret[0].(bool) + return ret0 +} + +// HADeployment indicates an expected call of HADeployment. +func (mr *MockLicenserMockRecorder) HADeployment() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HADeployment", reflect.TypeOf((*MockLicenser)(nil).HADeployment)) +} + +// MutualTLS mocks base method. +func (m *MockLicenser) MutualTLS() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MutualTLS") + ret0, _ := ret[0].(bool) + return ret0 +} + +// MutualTLS indicates an expected call of MutualTLS. +func (mr *MockLicenserMockRecorder) MutualTLS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MutualTLS", reflect.TypeOf((*MockLicenser)(nil).MutualTLS)) +} + +// PortalLinks mocks base method. +func (m *MockLicenser) PortalLinks() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PortalLinks") + ret0, _ := ret[0].(bool) + return ret0 +} + +// PortalLinks indicates an expected call of PortalLinks. +func (mr *MockLicenserMockRecorder) PortalLinks() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PortalLinks", reflect.TypeOf((*MockLicenser)(nil).PortalLinks)) +} + +// ProjectEnabled mocks base method. +func (m *MockLicenser) ProjectEnabled(projectID string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProjectEnabled", projectID) + ret0, _ := ret[0].(bool) + return ret0 +} + +// ProjectEnabled indicates an expected call of ProjectEnabled. +func (mr *MockLicenserMockRecorder) ProjectEnabled(projectID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProjectEnabled", reflect.TypeOf((*MockLicenser)(nil).ProjectEnabled), projectID) +} + +// RemoveEnabledProject mocks base method. +func (m *MockLicenser) RemoveEnabledProject(projectID string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RemoveEnabledProject", projectID) +} + +// RemoveEnabledProject indicates an expected call of RemoveEnabledProject. +func (mr *MockLicenserMockRecorder) RemoveEnabledProject(projectID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveEnabledProject", reflect.TypeOf((*MockLicenser)(nil).RemoveEnabledProject), projectID) +} + +// SynchronousWebhooks mocks base method. +func (m *MockLicenser) SynchronousWebhooks() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SynchronousWebhooks") + ret0, _ := ret[0].(bool) + return ret0 +} + +// SynchronousWebhooks indicates an expected call of SynchronousWebhooks. +func (mr *MockLicenserMockRecorder) SynchronousWebhooks() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SynchronousWebhooks", reflect.TypeOf((*MockLicenser)(nil).SynchronousWebhooks)) +} + +// Transformations mocks base method. +func (m *MockLicenser) Transformations() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Transformations") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Transformations indicates an expected call of Transformations. +func (mr *MockLicenserMockRecorder) Transformations() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Transformations", reflect.TypeOf((*MockLicenser)(nil).Transformations)) +} + +// UseForwardProxy mocks base method. +func (m *MockLicenser) UseForwardProxy() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UseForwardProxy") + ret0, _ := ret[0].(bool) + return ret0 +} + +// UseForwardProxy indicates an expected call of UseForwardProxy. +func (mr *MockLicenserMockRecorder) UseForwardProxy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UseForwardProxy", reflect.TypeOf((*MockLicenser)(nil).UseForwardProxy)) +} + +// WebhookAnalytics mocks base method. +func (m *MockLicenser) WebhookAnalytics() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WebhookAnalytics") + ret0, _ := ret[0].(bool) + return ret0 +} + +// WebhookAnalytics indicates an expected call of WebhookAnalytics. +func (mr *MockLicenserMockRecorder) WebhookAnalytics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WebhookAnalytics", reflect.TypeOf((*MockLicenser)(nil).WebhookAnalytics)) +} diff --git a/mocks/repository.go b/mocks/repository.go index 55c80ffea3..6351bd79f4 100644 --- a/mocks/repository.go +++ b/mocks/repository.go @@ -658,6 +658,21 @@ func (m *MockProjectRepository) EXPECT() *MockProjectRepositoryMockRecorder { return m.recorder } +// CountProjects mocks base method. +func (m *MockProjectRepository) CountProjects(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountProjects", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountProjects indicates an expected call of CountProjects. +func (mr *MockProjectRepositoryMockRecorder) CountProjects(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountProjects", reflect.TypeOf((*MockProjectRepository)(nil).CountProjects), ctx) +} + // CreateProject mocks base method. func (m *MockProjectRepository) CreateProject(arg0 context.Context, arg1 *datastore.Project) error { m.ctrl.T.Helper() @@ -782,6 +797,21 @@ func (m *MockOrganisationRepository) EXPECT() *MockOrganisationRepositoryMockRec return m.recorder } +// CountOrganisations mocks base method. +func (m *MockOrganisationRepository) CountOrganisations(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountOrganisations", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountOrganisations indicates an expected call of CountOrganisations. +func (mr *MockOrganisationRepositoryMockRecorder) CountOrganisations(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountOrganisations", reflect.TypeOf((*MockOrganisationRepository)(nil).CountOrganisations), ctx) +} + // CreateOrganisation mocks base method. func (m *MockOrganisationRepository) CreateOrganisation(arg0 context.Context, arg1 *datastore.Organisation) error { m.ctrl.T.Helper() @@ -2041,6 +2071,21 @@ func (m *MockUserRepository) EXPECT() *MockUserRepositoryMockRecorder { return m.recorder } +// CountUsers mocks base method. +func (m *MockUserRepository) CountUsers(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountUsers", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountUsers indicates an expected call of CountUsers. +func (mr *MockUserRepositoryMockRecorder) CountUsers(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountUsers", reflect.TypeOf((*MockUserRepository)(nil).CountUsers), ctx) +} + // CreateUser mocks base method. func (m *MockUserRepository) CreateUser(arg0 context.Context, arg1 *datastore.User) error { m.ctrl.T.Helper() @@ -2115,22 +2160,6 @@ func (mr *MockUserRepositoryMockRecorder) FindUserByToken(arg0, arg1 any) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindUserByToken", reflect.TypeOf((*MockUserRepository)(nil).FindUserByToken), arg0, arg1) } -// LoadUsersPaged mocks base method. -func (m *MockUserRepository) LoadUsersPaged(arg0 context.Context, arg1 datastore.Pageable) ([]datastore.User, datastore.PaginationData, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadUsersPaged", arg0, arg1) - ret0, _ := ret[0].([]datastore.User) - ret1, _ := ret[1].(datastore.PaginationData) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// LoadUsersPaged indicates an expected call of LoadUsersPaged. -func (mr *MockUserRepositoryMockRecorder) LoadUsersPaged(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadUsersPaged", reflect.TypeOf((*MockUserRepository)(nil).LoadUsersPaged), arg0, arg1) -} - // UpdateUser mocks base method. func (m *MockUserRepository) UpdateUser(ctx context.Context, user *datastore.User) error { m.ctrl.T.Helper() diff --git a/net/dispatcher.go b/net/dispatcher.go index 6077daddde..cb66d7edb4 100644 --- a/net/dispatcher.go +++ b/net/dispatcher.go @@ -11,6 +11,8 @@ import ( "net/url" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/pkg/httpheader" "github.com/frain-dev/convoy/pkg/log" @@ -21,7 +23,7 @@ type Dispatcher struct { client *http.Client } -func NewDispatcher(httpProxy string, enforceSecure bool) (*Dispatcher, error) { +func NewDispatcher(httpProxy string, licenser license.Licenser, enforceSecure bool) (*Dispatcher, error) { d := &Dispatcher{client: &http.Client{}} tr := &http.Transport{ @@ -32,13 +34,15 @@ func NewDispatcher(httpProxy string, enforceSecure bool) (*Dispatcher, error) { ExpectContinueTimeout: 1 * time.Second, } - proxyUrl, isValid, err := d.setProxy(httpProxy) - if err != nil { - return nil, err - } + if licenser.UseForwardProxy() { + proxyUrl, isValid, err := d.setProxy(httpProxy) + if err != nil { + return nil, err + } - if isValid { - tr.Proxy = http.ProxyURL(proxyUrl) + if isValid { + tr.Proxy = http.ProxyURL(proxyUrl) + } } // if enforceSecure is false, allow self-signed certificates, susceptible to MITM attacks. diff --git a/net/dispatcher_test.go b/net/dispatcher_test.go index c309b2d8bf..1af4788e69 100644 --- a/net/dispatcher_test.go +++ b/net/dispatcher_test.go @@ -9,6 +9,11 @@ import ( "testing" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + + "github.com/frain-dev/convoy/mocks" + "go.uber.org/mock/gomock" + "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/httpheader" "github.com/jarcoal/httpmock" @@ -294,3 +299,70 @@ func TestDispatcher_SendRequest(t *testing.T) { }) } } + +func TestNewDispatcher(t *testing.T) { + type args struct { + httpProxy string + enforceSecure bool + } + tests := []struct { + name string + args args + mockFn func(licenser license.Licenser) + wantProxy bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_set_proxy", + args: args{ + httpProxy: "https://21.3.32.33:443", + enforceSecure: false, + }, + mockFn: func(licenser license.Licenser) { + l := licenser.(*mocks.MockLicenser) + l.EXPECT().UseForwardProxy().Return(true) + }, + wantProxy: true, + wantErr: false, + wantErrMsg: "", + }, + { + name: "should_not_set_proxy", + args: args{ + httpProxy: "https://21.3.32.33:443", + enforceSecure: false, + }, + mockFn: func(licenser license.Licenser) { + l := licenser.(*mocks.MockLicenser) + l.EXPECT().UseForwardProxy().Return(false) + }, + wantProxy: false, + wantErr: false, + wantErrMsg: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + licenser := mocks.NewMockLicenser(ctrl) + if tt.mockFn != nil { + tt.mockFn(licenser) + } + d, err := NewDispatcher(tt.args.httpProxy, licenser, tt.args.enforceSecure) + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + + if tt.wantProxy { + require.NotNil(t, d.client.Transport.(*http.Transport).Proxy) + } + }) + } +} diff --git a/release.Dockerfile b/release.Dockerfile index b24afca2c3..9d644c2113 100644 --- a/release.Dockerfile +++ b/release.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.16.2 +FROM alpine:3.20.2 COPY convoy /cmd COPY configs/local/start.sh /start.sh diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 691b93846c..91f77f0bf5 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -13,3 +13,5 @@ export TEST_REDIS_HOST=localhost export TEST_REDIS_PORT=6379 make integration_tests + +make docker_e2e_tests diff --git a/scripts/ui.sh b/scripts/ui.sh index b112c5986c..1271b258cb 100755 --- a/scripts/ui.sh +++ b/scripts/ui.sh @@ -12,7 +12,7 @@ buildUi() { cd ./web/ui/dashboard || exit 1 # Install dependencies - npm ci + npm i # Run production build if [[ "$build" == "ce" ]]; then diff --git a/services/create_endpoint.go b/services/create_endpoint.go index 2a52656d67..36f0564628 100644 --- a/services/create_endpoint.go +++ b/services/create_endpoint.go @@ -6,6 +6,10 @@ import ( "net/http" "time" + "github.com/frain-dev/convoy" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/cache" "github.com/frain-dev/convoy/datastore" @@ -19,6 +23,7 @@ type CreateEndpointService struct { PortalLinkRepo datastore.PortalLinkRepository EndpointRepo datastore.EndpointRepository ProjectRepo datastore.ProjectRepository + Licenser license.Licenser E models.CreateEndpoint ProjectID string @@ -68,6 +73,14 @@ func (a *CreateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e UpdatedAt: time.Now(), } + if !a.Licenser.AdvancedEndpointMgmt() { + // switch to default timeout + endpoint.HttpTimeout = convoy.HTTP_TIMEOUT + + endpoint.SupportEmail = "" + endpoint.SlackWebhookURL = "" + } + if util.IsStringEmpty(endpoint.AppID) { endpoint.AppID = endpoint.UID } diff --git a/services/create_endpoint_test.go b/services/create_endpoint_test.go index 0117ffd6e8..de0b7352b4 100644 --- a/services/create_endpoint_test.go +++ b/services/create_endpoint_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/mocks" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -18,6 +20,7 @@ func provideCreateEndpointService(ctrl *gomock.Controller, e models.CreateEndpoi Cache: mocks.NewMockCache(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), ProjectRepo: mocks.NewMockProjectRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), E: e, ProjectID: projectID, } @@ -50,6 +53,7 @@ func TestCreateEndpointService_Run(t *testing.T) { SupportEmail: "endpoint@test.com", IsDisabled: false, SlackWebhookURL: "https://google.com", + HttpTimeout: 30, Secret: "1234", URL: "https://google.com", Description: "test_endpoint", @@ -63,7 +67,13 @@ func TestCreateEndpointService_Run(t *testing.T) { Return(project, nil) a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) - a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil) + a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Cond(func(x any) bool { + endpoint := x.(*datastore.Endpoint) + return endpoint.HttpTimeout == 30 + }), gomock.Any()).Times(1).Return(nil) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantEndpoint: &datastore.Endpoint{ Name: "endpoint", @@ -74,6 +84,56 @@ func TestCreateEndpointService_Run(t *testing.T) { {Value: "1234"}, }, AdvancedSignatures: true, + HttpTimeout: 30, + Url: "https://google.com", + Description: "test_endpoint", + RateLimit: 0, + Status: datastore.ActiveEndpointStatus, + RateLimitDuration: 0, + }, + wantErr: false, + }, + { + name: "should_default_http_timeout_endpoint_for_license_check_and_remove_slack_url_support_email", + args: args{ + ctx: ctx, + e: models.CreateEndpoint{ + Name: "endpoint", + SupportEmail: "endpoint@test.com", + IsDisabled: false, + SlackWebhookURL: "https://google.com", + Secret: "1234", + URL: "https://google.com", + HttpTimeout: 3, + Description: "test_endpoint", + }, + g: project, + }, + dbFn: func(app *CreateEndpointService) { + p, _ := app.ProjectRepo.(*mocks.MockProjectRepository) + p.EXPECT().FetchProjectByID(gomock.Any(), gomock.Any()). + Times(1). + Return(project, nil) + + a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) + a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Cond(func(x any) bool { + endpoint := x.(*datastore.Endpoint) + return endpoint.HttpTimeout == convoy.HTTP_TIMEOUT + }), gomock.Any()).Times(1).Return(nil) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) + }, + wantEndpoint: &datastore.Endpoint{ + Name: "endpoint", + SupportEmail: "", + SlackWebhookURL: "", + ProjectID: project.UID, + Secrets: []datastore.Secret{ + {Value: "1234"}, + }, + AdvancedSignatures: true, + HttpTimeout: convoy.HTTP_TIMEOUT, Url: "https://google.com", Description: "test_endpoint", RateLimit: 0, @@ -111,6 +171,9 @@ func TestCreateEndpointService_Run(t *testing.T) { a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantEndpoint: &datastore.Endpoint{ ProjectID: project.UID, @@ -156,6 +219,9 @@ func TestCreateEndpointService_Run(t *testing.T) { a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(errors.New("failed")) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantErr: true, wantErrMsg: "an error occurred while adding endpoint", diff --git a/services/create_organisation.go b/services/create_organisation.go index 33e595253a..3e4e7e8220 100644 --- a/services/create_organisation.go +++ b/services/create_organisation.go @@ -2,9 +2,12 @@ package services import ( "context" + "errors" "fmt" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/dchest/uniuri" "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/auth" @@ -21,10 +24,22 @@ type CreateOrganisationService struct { OrgMemberRepo datastore.OrganisationMemberRepository NewOrg *models.Organisation User *datastore.User + Licenser license.Licenser } +var ErrOrgLimit = errors.New("your instance has reached it's organisation limit, upgrade to create new organisations") + func (co *CreateOrganisationService) Run(ctx context.Context) (*datastore.Organisation, error) { - err := util.Validate(co.NewOrg) + ok, err := co.Licenser.CreateOrg(ctx) + if err != nil { + return nil, &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return nil, &ServiceError{ErrMsg: ErrOrgLimit.Error(), Err: ErrOrgLimit} + } + + err = util.Validate(co.NewOrg) if err != nil { return nil, &ServiceError{ErrMsg: err.Error()} } diff --git a/services/create_organisation_test.go b/services/create_organisation_test.go index e80458291a..ee61979b5e 100644 --- a/services/create_organisation_test.go +++ b/services/create_organisation_test.go @@ -17,6 +17,7 @@ func provideCreateOrganisationService(ctrl *gomock.Controller, newOrg *models.Or return &CreateOrganisationService{ OrgRepo: mocks.NewMockOrganisationRepository(ctrl), OrgMemberRepo: mocks.NewMockOrganisationMemberRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), NewOrg: newOrg, User: user, } @@ -53,6 +54,9 @@ func TestCreateOrganisationService_Run(t *testing.T) { om, _ := os.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -63,6 +67,10 @@ func TestCreateOrganisationService_Run(t *testing.T) { newOrg: &models.Organisation{Name: ""}, user: &datastore.User{UID: "1234"}, }, + dbFn: func(os *CreateOrganisationService) { + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + }, wantErr: true, wantErrMsg: "organisation name is required", }, @@ -74,6 +82,9 @@ func TestCreateOrganisationService_Run(t *testing.T) { user: &datastore.User{UID: "1234"}, }, dbFn: func(os *CreateOrganisationService) { + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + a, _ := os.OrgRepo.(*mocks.MockOrganisationRepository) a.EXPECT().CreateOrganisation(gomock.Any(), gomock.Any()). Times(1).Return(errors.New("failed")) @@ -81,6 +92,20 @@ func TestCreateOrganisationService_Run(t *testing.T) { wantErr: true, wantErrMsg: "failed to create organisation", }, + { + name: "should_fail_to_create_organisation_for_license_check", + args: args{ + ctx: ctx, + newOrg: &models.Organisation{Name: "new_org"}, + user: &datastore.User{UID: "1234"}, + }, + dbFn: func(os *CreateOrganisationService) { + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrMsg: ErrOrgLimit.Error(), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/services/create_subscription.go b/services/create_subscription.go index 1b6a30cd9d..76c749c04c 100644 --- a/services/create_subscription.go +++ b/services/create_subscription.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "errors" - "gopkg.in/guregu/null.v4" "net/http" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/oklog/ulid/v2" "github.com/frain-dev/convoy/api/models" @@ -27,6 +29,7 @@ type CreateSubscriptionService struct { SourceRepo datastore.SourceRepository Project *datastore.Project NewSubscription *models.CreateSubscription + Licenser license.Licenser } func (s *CreateSubscriptionService) Run(ctx context.Context) (*datastore.Subscription, error) { @@ -72,22 +75,28 @@ func (s *CreateSubscriptionService) Run(ctx context.Context) (*datastore.Subscri Type: datastore.SubscriptionTypeAPI, SourceID: s.NewSubscription.SourceID, EndpointID: s.NewSubscription.EndpointID, - Function: null.StringFrom(s.NewSubscription.Function), RetryConfig: retryConfig, AlertConfig: s.NewSubscription.AlertConfig.Transform(), - FilterConfig: s.NewSubscription.FilterConfig.Transform(), RateLimitConfig: s.NewSubscription.RateLimitConfig.Transform(), CreatedAt: time.Now(), UpdatedAt: time.Now(), } + if s.Licenser.AdvancedSubscriptions() { + subscription.FilterConfig = s.NewSubscription.FilterConfig.Transform() + } + + if s.Licenser.Transformations() { + subscription.Function = null.StringFrom(s.NewSubscription.Function) + } + if subscription.FilterConfig == nil { subscription.FilterConfig = &datastore.FilterConfiguration{} } - if subscription.FilterConfig.EventTypes == nil || len(subscription.FilterConfig.EventTypes) == 0 { + if len(subscription.FilterConfig.EventTypes) == 0 { subscription.FilterConfig.EventTypes = []string{"*"} } diff --git a/services/create_subscription_test.go b/services/create_subscription_test.go index d9b3884243..5525a8ec12 100644 --- a/services/create_subscription_test.go +++ b/services/create_subscription_test.go @@ -3,7 +3,11 @@ package services import ( "context" "errors" + "reflect" "testing" + "time" + + "gopkg.in/guregu/null.v4" "github.com/frain-dev/convoy/mocks" "github.com/stretchr/testify/require" @@ -18,6 +22,7 @@ func provideCreateSubscriptionService(ctrl *gomock.Controller, project *datastor SubRepo: mocks.NewMockSubscriptionRepository(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), SourceRepo: mocks.NewMockSourceRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), Project: project, NewSubscription: newSub, } @@ -58,6 +63,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { EndpointID: "endpoint-id-1", }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -78,6 +87,79 @@ func TestCreateSubscriptionService_Run(t *testing.T) { ) }, }, + { + name: "should skip filter config & function fields for subscription for outgoing project", + args: args{ + ctx: ctx, + newSubscription: &models.CreateSubscription{ + Name: "sub 1", + SourceID: "source-id-1", + EndpointID: "endpoint-id-1", + Function: "console.log", + FilterConfig: &models.FilterConfiguration{ + EventTypes: []string{"invoice.created"}, + Filter: models.FS{ + Headers: datastore.M{"x-msg-type": "stream-data"}, + Body: datastore.M{"offset": "1234"}, + }, + }, + }, + project: &datastore.Project{UID: "12345", Type: datastore.OutgoingProject, Config: &datastore.ProjectConfig{MultipleEndpointSubscriptions: false}}, + }, + wantSubscription: &datastore.Subscription{ + Name: "sub 1", + Type: datastore.SubscriptionTypeAPI, + SourceID: "source-id-1", + EndpointID: "endpoint-id-1", + }, + dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(false) + licenser.EXPECT().Transformations().Times(1).Return(false) + + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) + s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Cond(func(x any) bool { + sub := x.(*datastore.Subscription) + var uid string + uid, sub.UID = sub.UID, "" + sub.CreatedAt, sub.UpdatedAt = time.Time{}, time.Time{} + + c := &datastore.Subscription{ + Name: "sub 1", + SourceID: "source-id-1", + EndpointID: "endpoint-id-1", + ProjectID: "12345", + Function: null.String{}, + Type: datastore.SubscriptionTypeAPI, + FilterConfig: &datastore.FilterConfiguration{ + EventTypes: []string{"*"}, + Filter: datastore.FilterSchema{ + Headers: datastore.M{}, + Body: datastore.M{}, + }, + }, + } + + ok := reflect.DeepEqual(sub, c) + sub.UID = uid + return ok + })).Times(1).Return(nil) + + s.EXPECT().CountEndpointSubscriptions(gomock.Any(), "12345", "endpoint-id-1"). + Times(1). + Return(int64(0), nil) + + a, _ := ss.EndpointRepo.(*mocks.MockEndpointRepository) + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()). + Times(1).Return( + &datastore.Endpoint{ + UID: "endpoint-id-1", + ProjectID: "12345", + }, + nil, + ) + }, + }, { name: "should fail to count endpoint subscriptions for outgoing project if multi endpoints for subscriptions is false", args: args{ @@ -132,6 +214,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { EndpointID: "endpoint-id-1", }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -202,6 +288,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { EndpointID: "endpoint-id-1", }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -355,6 +445,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { }, }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -393,6 +487,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { }, }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). diff --git a/services/invite_user.go b/services/invite_user.go index 08661ecdb7..a6c1660b37 100644 --- a/services/invite_user.go +++ b/services/invite_user.go @@ -3,10 +3,12 @@ package services import ( "context" "fmt" - "github.com/frain-dev/convoy/pkg/msgpack" "strings" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/pkg/msgpack" + "github.com/dchest/uniuri" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth" @@ -26,9 +28,19 @@ type InviteUserService struct { Role auth.Role User *datastore.User Organisation *datastore.Organisation + Licenser license.Licenser } func (iu *InviteUserService) Run(ctx context.Context) (*datastore.OrganisationInvite, error) { + ok, err := iu.Licenser.CreateUser(ctx) + if err != nil { + return nil, &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return nil, &ServiceError{ErrMsg: ErrUserLimit.Error()} + } + iv := &datastore.OrganisationInvite{ UID: ulid.Make().String(), OrganisationID: iu.Organisation.UID, @@ -41,7 +53,7 @@ func (iu *InviteUserService) Run(ctx context.Context) (*datastore.OrganisationIn UpdatedAt: time.Now(), } - err := iu.InviteRepo.CreateOrganisationInvite(ctx, iv) + err = iu.InviteRepo.CreateOrganisationInvite(ctx, iv) if err != nil { errMsg := "failed to invite member" log.FromContext(ctx).WithError(err).Error(errMsg) diff --git a/services/invite_user_test.go b/services/invite_user_test.go index 03df075918..20a82207d2 100644 --- a/services/invite_user_test.go +++ b/services/invite_user_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/auth" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" @@ -18,6 +20,7 @@ func TestInviteUserService(t *testing.T) { type args struct { inviteRepo datastore.OrganisationInviteRepository queue queue.Queuer + Licenser license.Licenser } dbErr := errors.New("failed to create invite") @@ -37,6 +40,9 @@ func TestInviteUserService(t *testing.T) { user: &datastore.User{}, organisation: &datastore.Organisation{}, mockDep: func(a args) { + licenser, _ := a.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + ivRepo, _ := a.inviteRepo.(*mocks.MockOrganisationInviteRepository) ivRepo.EXPECT().CreateOrganisationInvite( gomock.Any(), @@ -54,6 +60,9 @@ func TestInviteUserService(t *testing.T) { organisation: &datastore.Organisation{}, err: dbErr, mockDep: func(a args) { + licenser, _ := a.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + ivRepo, _ := a.inviteRepo.(*mocks.MockOrganisationInviteRepository) ivRepo.EXPECT().CreateOrganisationInvite( gomock.Any(), @@ -61,6 +70,17 @@ func TestInviteUserService(t *testing.T) { ).Return(dbErr) }, }, + { + name: "should_fail_to_invite_user_for_license_check", + inviteeEmail: "sidemen@default.com", + user: &datastore.User{}, + organisation: &datastore.Organisation{}, + err: ErrUserLimit, + mockDep: func(a args) { + licenser, _ := a.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(false, nil) + }, + }, } for _, tt := range tests { @@ -74,6 +94,7 @@ func TestInviteUserService(t *testing.T) { args := args{ inviteRepo: mocks.NewMockOrganisationInviteRepository(ctrl), queue: mocks.NewMockQueuer(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), } if tt.mockDep != nil { @@ -86,6 +107,7 @@ func TestInviteUserService(t *testing.T) { InviteeEmail: tt.inviteeEmail, User: tt.user, Organisation: tt.organisation, + Licenser: args.Licenser, Role: tt.role, } diff --git a/services/process_invite.go b/services/process_invite.go index 9ab0d79d6e..9b0b02b7aa 100644 --- a/services/process_invite.go +++ b/services/process_invite.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/api/models" "github.com/oklog/ulid/v2" @@ -21,13 +23,25 @@ type ProcessInviteService struct { UserRepo datastore.UserRepository OrgRepo datastore.OrganisationRepository OrgMemberRepo datastore.OrganisationMemberRepository + Licenser license.Licenser Token string Accepted bool NewUser *models.User } +var ErrUserLimit = errors.New("your instance has reached it's user limit, upgrade to add new users") + func (pis *ProcessInviteService) Run(ctx context.Context) error { + ok, err := pis.Licenser.CreateUser(ctx) + if err != nil { + return &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return &ServiceError{ErrMsg: ErrUserLimit.Error()} + } + iv, err := pis.InviteRepo.FetchOrganisationInviteByToken(ctx, pis.Token) if err != nil { log.FromContext(ctx).WithError(err).Error("failed to fetch organisation member invite by token and email") diff --git a/services/process_invite_test.go b/services/process_invite_test.go index 4d297ace2c..b994318d7d 100644 --- a/services/process_invite_test.go +++ b/services/process_invite_test.go @@ -23,6 +23,7 @@ func provideProcessInviteService(ctrl *gomock.Controller, token string, accepted UserRepo: mocks.NewMockUserRepository(ctrl), OrgRepo: mocks.NewMockOrganisationRepository(ctrl), OrgMemberRepo: mocks.NewMockOrganisationMemberRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), Token: token, Accepted: accepted, @@ -100,6 +101,9 @@ func TestProcessInviteService_Run(t *testing.T) { om, _ := pis.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -128,10 +132,29 @@ func TestProcessInviteService_Run(t *testing.T) { }, nil, ) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "organisation member invite already accepted", }, + + { + name: "should_error_for_licence_cant_user", + args: args{ + ctx: ctx, + token: "abcdef", + accepted: true, + newUser: nil, + }, + dbFn: func(pis *ProcessInviteService) { + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrMsg: ErrUserLimit.Error(), + }, { name: "should_error_for_invite_already_declined", args: args{ @@ -156,6 +179,9 @@ func TestProcessInviteService_Run(t *testing.T) { }, nil, ) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "organisation member invite already declined", @@ -185,6 +211,9 @@ func TestProcessInviteService_Run(t *testing.T) { }, nil, ) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "organisation member invite already expired", @@ -201,6 +230,9 @@ func TestProcessInviteService_Run(t *testing.T) { oir, _ := pis.InviteRepo.(*mocks.MockOrganisationInviteRepository) oir.EXPECT().FetchOrganisationInviteByToken(gomock.Any(), "abcdef"). Times(1).Return(nil, errors.New("failed")) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to fetch organisation member invite", @@ -241,6 +273,8 @@ func TestProcessInviteService_Run(t *testing.T) { Endpoint: "", }, }).Times(1).Return(nil) + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -272,6 +306,9 @@ func TestProcessInviteService_Run(t *testing.T) { u, _ := pis.UserRepo.(*mocks.MockUserRepository) u.EXPECT().FindUserByEmail(gomock.Any(), "test@email.com"). Times(1).Return(nil, errors.New("failed")) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to find user by email", @@ -332,6 +369,9 @@ func TestProcessInviteService_Run(t *testing.T) { om, _ := pis.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -363,6 +403,9 @@ func TestProcessInviteService_Run(t *testing.T) { u, _ := pis.UserRepo.(*mocks.MockUserRepository) u.EXPECT().FindUserByEmail(gomock.Any(), "test@email.com"). Times(1).Return(nil, datastore.ErrUserNotFound) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "new user is nil", @@ -401,6 +444,9 @@ func TestProcessInviteService_Run(t *testing.T) { u, _ := pis.UserRepo.(*mocks.MockUserRepository) u.EXPECT().FindUserByEmail(gomock.Any(), "test@email.com"). Times(1).Return(nil, datastore.ErrUserNotFound) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "first_name:please provide a first name", @@ -441,6 +487,9 @@ func TestProcessInviteService_Run(t *testing.T) { Times(1).Return(nil, datastore.ErrUserNotFound) u.EXPECT().CreateUser(gomock.Any(), gomock.Any()).Times(1).Return(errors.New("failed")) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to create user", @@ -482,6 +531,8 @@ func TestProcessInviteService_Run(t *testing.T) { o, _ := pis.OrgRepo.(*mocks.MockOrganisationRepository) o.EXPECT().FetchOrganisationByID(gomock.Any(), "123ab"). Times(1).Return(nil, errors.New("failed")) + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to fetch organisation by id", @@ -588,6 +639,9 @@ func TestProcessInviteService_Run(t *testing.T) { om, _ := pis.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to update accepted organisation invite", diff --git a/services/project_service.go b/services/project_service.go index e9bc41c761..0370498baa 100644 --- a/services/project_service.go +++ b/services/project_service.go @@ -7,6 +7,8 @@ import ( "net/http" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/auth" "github.com/oklog/ulid/v2" @@ -22,20 +24,33 @@ type ProjectService struct { projectRepo datastore.ProjectRepository eventRepo datastore.EventRepository eventDeliveryRepo datastore.EventDeliveryRepository + Licenser license.Licenser cache cache.Cache } -func NewProjectService(apiKeyRepo datastore.APIKeyRepository, projectRepo datastore.ProjectRepository, eventRepo datastore.EventRepository, eventDeliveryRepo datastore.EventDeliveryRepository, cache cache.Cache) (*ProjectService, error) { +func NewProjectService(apiKeyRepo datastore.APIKeyRepository, projectRepo datastore.ProjectRepository, eventRepo datastore.EventRepository, eventDeliveryRepo datastore.EventDeliveryRepository, licenser license.Licenser, cache cache.Cache) (*ProjectService, error) { return &ProjectService{ apiKeyRepo: apiKeyRepo, projectRepo: projectRepo, eventRepo: eventRepo, eventDeliveryRepo: eventDeliveryRepo, + Licenser: licenser, cache: cache, }, nil } +var ErrProjectLimit = errors.New("your instance has reached it's project limit, upgrade to create more projects") + func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models.CreateProject, org *datastore.Organisation, member *datastore.OrganisationMember) (*datastore.Project, *models.APIKeyResponse, error) { + ok, err := ps.Licenser.CreateProject(ctx) + if err != nil { + return nil, nil, util.NewServiceError(http.StatusBadRequest, err) + } + + if !ok { + return nil, nil, util.NewServiceError(http.StatusBadRequest, ErrProjectLimit) + } + projectName := newProject.Name projectConfig := newProject.Config.Transform() @@ -73,6 +88,10 @@ func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models. } } + if !ps.Licenser.AdvancedWebhookFiltering() { + projectConfig.SearchPolicy = "" + } + project := &datastore.Project{ UID: ulid.Make().String(), Name: projectName, @@ -84,7 +103,7 @@ func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models. UpdatedAt: time.Now(), } - err := ps.projectRepo.CreateProject(ctx, project) + err = ps.projectRepo.CreateProject(ctx, project) if err != nil { log.FromContext(ctx).WithError(err).Error("failed to create project") if errors.Is(err, datastore.ErrDuplicateProjectName) { @@ -129,6 +148,11 @@ func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models. Key: keyString, } + // if this is a community license, add this project to list of enabled projects + // because if the initial license check above passed, then the project count limit had + // not been reached + ps.Licenser.AddEnabledProject(project.UID) + return project, resp, nil } @@ -157,6 +181,10 @@ func (ps *ProjectService) UpdateProject(ctx context.Context, project *datastore. project.LogoURL = update.LogoURL } + if !ps.Licenser.AdvancedWebhookFiltering() { + project.Config.SearchPolicy = "" + } + err := ps.projectRepo.UpdateProject(ctx, project) if err != nil { log.FromContext(ctx).WithError(err).Error("failed to to update project") diff --git a/services/project_service_test.go b/services/project_service_test.go index 5b59b4f418..f76ed1df11 100644 --- a/services/project_service_test.go +++ b/services/project_service_test.go @@ -21,9 +21,10 @@ func provideProjectService(ctrl *gomock.Controller) (*ProjectService, error) { eventRepo := mocks.NewMockEventRepository(ctrl) eventDeliveryRepo := mocks.NewMockEventDeliveryRepository(ctrl) apiKeyRepo := mocks.NewMockAPIKeyRepository(ctrl) + l := mocks.NewMockLicenser(ctrl) cache := mocks.NewMockCache(ctrl) - return NewProjectService(apiKeyRepo, projectRepo, eventRepo, eventDeliveryRepo, cache) + return NewProjectService(apiKeyRepo, projectRepo, eventRepo, eventDeliveryRepo, l, cache) } func TestProjectService_CreateProject(t *testing.T) { @@ -55,6 +56,7 @@ func TestProjectService_CreateProject(t *testing.T) { Signature: &models.SignatureConfiguration{ Header: "X-Convoy-Signature", }, + SearchPolicy: "300h", Strategy: &models.StrategyConfiguration{ Type: "linear", Duration: 20, @@ -83,6 +85,11 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project", @@ -98,7 +105,8 @@ func TestProjectService_CreateProject(t *testing.T) { Duration: 20, RetryCount: 4, }, - SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: true}, + SearchPolicy: "300h", + SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: true}, RateLimit: &datastore.RateLimitConfiguration{ Count: 1000, Duration: 60, @@ -121,6 +129,7 @@ func TestProjectService_CreateProject(t *testing.T) { Signature: &models.SignatureConfiguration{ Header: "X-Convoy-Signature", }, + SearchPolicy: "300h", Strategy: &models.StrategyConfiguration{ Type: "linear", Duration: 20, @@ -150,6 +159,11 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project", @@ -160,7 +174,8 @@ func TestProjectService_CreateProject(t *testing.T) { Signature: &datastore.SignatureConfiguration{ Header: "X-Convoy-Signature", }, - SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: false}, + SearchPolicy: "300h", + SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: false}, Strategy: &datastore.StrategyConfiguration{ Type: "linear", Duration: 20, @@ -202,6 +217,11 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project_1", @@ -255,6 +275,11 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project", @@ -282,6 +307,79 @@ func TestProjectService_CreateProject(t *testing.T) { }, wantErr: false, }, + { + name: "should_remove_search_policy_for_license_check", + args: args{ + ctx: ctx, + newProject: &models.CreateProject{ + Name: "test_project", + Type: "outgoing", + LogoURL: "https://google.com", + Config: &models.ProjectConfig{ + Signature: &models.SignatureConfiguration{ + Header: "X-Convoy-Signature", + }, + SearchPolicy: "300h", + Strategy: &models.StrategyConfiguration{ + Type: "linear", + Duration: 20, + RetryCount: 4, + }, + RateLimit: &models.RateLimitConfiguration{ + Count: 1000, + Duration: 60, + }, + ReplayAttacks: true, + }, + }, + org: &datastore.Organisation{UID: "1234"}, + member: &datastore.OrganisationMember{ + UID: "abc", + OrganisationID: "1234", + Role: auth.Role{Type: auth.RoleSuperUser}, + }, + }, + dbFn: func(gs *ProjectService) { + a, _ := gs.projectRepo.(*mocks.MockProjectRepository) + a.EXPECT().CreateProject(gomock.Any(), gomock.Any()). + Times(1).Return(nil) + + a.EXPECT().FetchProjectByID(gomock.Any(), gomock.Any()).Times(1).Return(&datastore.Project{UID: "abc", OrganisationID: "1234"}, nil) + + apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) + apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(false) + }, + wantProject: &datastore.Project{ + Name: "test_project", + Type: "outgoing", + LogoURL: "https://google.com", + OrganisationID: "1234", + Config: &datastore.ProjectConfig{ + Signature: &datastore.SignatureConfiguration{ + Header: "X-Convoy-Signature", + }, + Strategy: &datastore.StrategyConfiguration{ + Type: "linear", + Duration: 20, + RetryCount: 4, + }, + SearchPolicy: "", + SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: true}, + RateLimit: &datastore.RateLimitConfiguration{ + Count: 1000, + Duration: 60, + }, + // RetentionPolicy: &datastore.DefaultRetentionPolicy, + ReplayAttacks: true, + }, + }, + wantErr: false, + }, { name: "should_fail_to_create_project", args: args{ @@ -309,6 +407,10 @@ func TestProjectService_CreateProject(t *testing.T) { a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().CreateProject(gomock.Any(), gomock.Any()). Times(1).Return(errors.New("failed")) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantErr: true, wantErrCode: http.StatusBadRequest, @@ -380,11 +482,46 @@ func TestProjectService_CreateProject(t *testing.T) { a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().CreateProject(gomock.Any(), gomock.Any()). Times(1).Return(datastore.ErrDuplicateProjectName) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantErr: true, wantErrCode: http.StatusBadRequest, wantErrMsg: "a project with this name already exists", }, + { + name: "should_error_for_cant_create_project", + args: args{ + ctx: ctx, + newProject: &models.CreateProject{ + Name: "test_project", + Type: "incoming", + LogoURL: "https://google.com", + Config: &models.ProjectConfig{ + Signature: &models.SignatureConfiguration{ + Header: "X-Convoy-Signature", + }, + Strategy: &models.StrategyConfiguration{ + Type: "linear", + Duration: 20, + RetryCount: 4, + }, + ReplayAttacks: true, + }, + }, + org: &datastore.Organisation{UID: "1234"}, + member: &datastore.OrganisationMember{}, + }, + dbFn: func(gs *ProjectService) { + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrCode: http.StatusBadRequest, + wantErrMsg: ErrProjectLimit.Error(), + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -509,6 +646,9 @@ func TestProjectService_UpdateProject(t *testing.T) { }, }, dbFn: func(gs *ProjectService) { + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) + a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().UpdateProject(gomock.Any(), gomock.Any()).Times(1).Return(nil) }, @@ -536,6 +676,9 @@ func TestProjectService_UpdateProject(t *testing.T) { }, }, dbFn: func(gs *ProjectService) { + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) + a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().UpdateProject(gomock.Any(), gomock.Any()).Times(1).Return(errors.New("failed")) }, diff --git a/services/register_user.go b/services/register_user.go index 0ca3ccc56a..aa989cd585 100644 --- a/services/register_user.go +++ b/services/register_user.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" - "github.com/frain-dev/convoy/pkg/msgpack" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/pkg/msgpack" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/internal/email" "github.com/frain-dev/convoy/pkg/log" @@ -26,12 +28,22 @@ type RegisterUserService struct { Queue queue.Queuer JWT *jwt.Jwt ConfigRepo datastore.ConfigurationRepository + Licenser license.Licenser BaseURL string Data *models.RegisterUser } func (u *RegisterUserService) Run(ctx context.Context) (*datastore.User, *jwt.Token, error) { + ok, err := u.Licenser.CreateUser(ctx) + if err != nil { + return nil, nil, &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return nil, nil, &ServiceError{ErrMsg: ErrUserLimit.Error()} + } + config, err := u.ConfigRepo.LoadConfiguration(ctx) if err != nil && !errors.Is(err, datastore.ErrConfigNotFound) { return nil, nil, &ServiceError{ErrMsg: "failed to load configuration", Err: err} @@ -77,13 +89,16 @@ func (u *RegisterUserService) Run(ctx context.Context) (*datastore.User, *jwt.To co := CreateOrganisationService{ OrgRepo: u.OrgRepo, OrgMemberRepo: u.OrgMemberRepo, + Licenser: u.Licenser, NewOrg: &models.Organisation{Name: u.Data.OrganisationName}, User: user, } _, err = co.Run(ctx) if err != nil { - return nil, nil, err + if !errors.Is(err, ErrOrgLimit) && !errors.Is(err, ErrUserLimit) { + return nil, nil, err + } } token, err := u.JWT.GenerateToken(user) diff --git a/services/register_user_test.go b/services/register_user_test.go index 63604d6060..9cb6b87576 100644 --- a/services/register_user_test.go +++ b/services/register_user_test.go @@ -25,6 +25,7 @@ func provideRegisterUserService(ctrl *gomock.Controller, t *testing.T, baseUrl s OrgRepo: mocks.NewMockOrganisationRepository(ctrl), OrgMemberRepo: mocks.NewMockOrganisationMemberRepository(ctrl), Queue: mocks.NewMockQueuer(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), JWT: jwt.NewJwt(&configuration.Auth.Jwt, c), ConfigRepo: mocks.NewMockConfigurationRepository(ctrl), BaseURL: baseUrl, @@ -86,8 +87,39 @@ func TestRegisterUserService_Run(t *testing.T) { orgMemberRepo.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) queue.EXPECT().Write(gomock.Any(), gomock.Any(), gomock.Any()) + + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, }, + { + name: "should_not_register_user_when_cannot_create_user", + wantConfig: true, + args: args{ + ctx: ctx, + user: &models.RegisterUser{ + FirstName: "test", + LastName: "test", + Email: "test@test.com", + Password: "123456", + OrganisationName: "test", + }, + }, + wantUser: &datastore.User{ + UID: "12345", + FirstName: "test", + LastName: "test", + Email: "test@test.com", + }, + dbFn: func(u *RegisterUserService) { + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrMsg: ErrUserLimit.Error(), + }, + { name: "should_fail_to_load_config", wantConfig: true, @@ -102,6 +134,9 @@ func TestRegisterUserService_Run(t *testing.T) { }, }, dbFn: func(u *RegisterUserService) { + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + configRepo, _ := u.ConfigRepo.(*mocks.MockConfigurationRepository) configRepo.EXPECT().LoadConfiguration(gomock.Any()).Times(1).Return(nil, errors.New("failed")) }, @@ -142,6 +177,10 @@ func TestRegisterUserService_Run(t *testing.T) { orgMemberRepo.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) queue.EXPECT().Write(gomock.Any(), gomock.Any(), gomock.Any()) + + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, }, { @@ -158,6 +197,9 @@ func TestRegisterUserService_Run(t *testing.T) { }, }, dbFn: func(u *RegisterUserService) { + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + configRepo, _ := u.ConfigRepo.(*mocks.MockConfigurationRepository) configRepo.EXPECT().LoadConfiguration(gomock.Any()).Times(1).Return(&datastore.Configuration{ UID: "12345", diff --git a/services/update_endpoint.go b/services/update_endpoint.go index 001a3aed9f..431b6dbafa 100644 --- a/services/update_endpoint.go +++ b/services/update_endpoint.go @@ -4,6 +4,10 @@ import ( "context" "time" + "github.com/frain-dev/convoy" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/api/models" @@ -16,6 +20,7 @@ type UpdateEndpointService struct { Cache cache.Cache EndpointRepo datastore.EndpointRepository ProjectRepo datastore.ProjectRepository + Licenser license.Licenser E models.UpdateEndpoint Endpoint *datastore.Endpoint @@ -37,7 +42,7 @@ func (a *UpdateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e return nil, &ServiceError{ErrMsg: err.Error()} } - endpoint, err = updateEndpoint(endpoint, a.E, a.Project) + endpoint, err = a.updateEndpoint(endpoint, a.E, a.Project) if err != nil { return nil, &ServiceError{ErrMsg: err.Error()} } @@ -52,17 +57,17 @@ func (a *UpdateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e return endpoint, nil } -func updateEndpoint(endpoint *datastore.Endpoint, e models.UpdateEndpoint, project *datastore.Project) (*datastore.Endpoint, error) { +func (a *UpdateEndpointService) updateEndpoint(endpoint *datastore.Endpoint, e models.UpdateEndpoint, project *datastore.Project) (*datastore.Endpoint, error) { endpoint.Url = e.URL endpoint.Description = e.Description endpoint.Name = *e.Name - if e.SupportEmail != nil { + if e.SupportEmail != nil && a.Licenser.AdvancedEndpointMgmt() { endpoint.SupportEmail = *e.SupportEmail } - if e.SlackWebhookURL != nil { + if e.SlackWebhookURL != nil && a.Licenser.AdvancedEndpointMgmt() { endpoint.SlackWebhookURL = *e.SlackWebhookURL } @@ -80,6 +85,11 @@ func updateEndpoint(endpoint *datastore.Endpoint, e models.UpdateEndpoint, proje if e.HttpTimeout != 0 { endpoint.HttpTimeout = e.HttpTimeout + + if !a.Licenser.AdvancedEndpointMgmt() { + // switch to default timeout + endpoint.HttpTimeout = convoy.HTTP_TIMEOUT + } } if !util.IsStringEmpty(e.OwnerID) { diff --git a/services/update_endpoint_test.go b/services/update_endpoint_test.go index 0f5a452ad6..58d6801246 100644 --- a/services/update_endpoint_test.go +++ b/services/update_endpoint_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/mocks" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -18,6 +20,7 @@ func provideUpdateEndpointService(ctrl *gomock.Controller, e models.UpdateEndpoi Cache: mocks.NewMockCache(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), ProjectRepo: mocks.NewMockProjectRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), E: e, Endpoint: Endpoint, Project: Project, @@ -71,6 +74,9 @@ func TestUpdateEndpointService_Run(t *testing.T) { a.EXPECT().UpdateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()). Times(1).Return(nil) + + licenser, _ := as.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantErr: false, }, @@ -96,10 +102,51 @@ func TestUpdateEndpointService_Run(t *testing.T) { a.EXPECT().UpdateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()). Times(1).Return(errors.New("failed")) + + licenser, _ := as.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantErr: true, wantErrMsg: "an error occurred while updating endpoints", }, + { + name: "should_default_endpoint_http_timeout_for_license_check_failed", + args: args{ + ctx: ctx, + e: models.UpdateEndpoint{ + Name: stringPtr("Endpoint2"), + Description: "test_endpoint", + URL: "https://www.google.com/webhp", + RateLimit: 10000, + RateLimitDuration: 60, + HttpTimeout: 200, + }, + endpoint: &datastore.Endpoint{UID: "endpoint2"}, + project: project, + }, + wantEndpoint: &datastore.Endpoint{ + Name: "Endpoint2", + Description: "test_endpoint", + Url: "https://www.google.com/webhp", + RateLimit: 10000, + RateLimitDuration: 60, + HttpTimeout: convoy.HTTP_TIMEOUT, + }, + dbFn: func(as *UpdateEndpointService) { + a, _ := as.EndpointRepo.(*mocks.MockEndpointRepository) + a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), "1234567890"). + Times(1).Return(&datastore.Endpoint{UID: "endpoint2"}, nil) + + a.EXPECT().UpdateEndpoint(gomock.Any(), gomock.Cond(func(x any) bool { + endpoint := x.(*datastore.Endpoint) + return endpoint.HttpTimeout == convoy.HTTP_TIMEOUT + }), gomock.Any()).Times(1).Return(nil) + + licenser, _ := as.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) + }, + wantErr: false, + }, { name: "should_error_for_endpoint_not_found", args: args{ diff --git a/services/update_subscription.go b/services/update_subscription.go index c94a41e343..236323d980 100644 --- a/services/update_subscription.go +++ b/services/update_subscription.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "errors" + + "github.com/frain-dev/convoy/internal/pkg/license" "gopkg.in/guregu/null.v4" "github.com/frain-dev/convoy/api/models" @@ -18,9 +20,11 @@ var ( ) type UpdateSubscriptionService struct { - SubRepo datastore.SubscriptionRepository - EndpointRepo datastore.EndpointRepository - SourceRepo datastore.SourceRepository + SubRepo datastore.SubscriptionRepository + EndpointRepo datastore.EndpointRepository + SourceRepo datastore.SourceRepository + Licenser license.Licenser + ProjectId string SubscriptionId string Update *models.UpdateSubscription @@ -46,7 +50,7 @@ func (s *UpdateSubscriptionService) Run(ctx context.Context) (*datastore.Subscri subscription.SourceID = s.Update.SourceID } - if !util.IsStringEmpty(s.Update.Function) { + if !util.IsStringEmpty(s.Update.Function) && s.Licenser.Transformations() { subscription.Function = null.StringFrom(s.Update.Function) } @@ -102,7 +106,7 @@ func (s *UpdateSubscriptionService) Run(ctx context.Context) (*datastore.Subscri subscription.RetryConfig.RetryCount = s.Update.RetryConfig.RetryCount } - if s.Update.FilterConfig != nil { + if s.Update.FilterConfig != nil && s.Licenser.AdvancedSubscriptions() { if len(s.Update.FilterConfig.EventTypes) > 0 { subscription.FilterConfig.EventTypes = s.Update.FilterConfig.EventTypes } diff --git a/services/update_subscription_test.go b/services/update_subscription_test.go index 64fc5c4455..0dcdfb942a 100644 --- a/services/update_subscription_test.go +++ b/services/update_subscription_test.go @@ -18,6 +18,7 @@ func provideUpdateSubscriptionService(ctrl *gomock.Controller, projectID string, SubRepo: mocks.NewMockSubscriptionRepository(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), SourceRepo: mocks.NewMockSourceRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), ProjectId: projectID, SubscriptionId: subID, Update: update, diff --git a/sql/1724236900.sql b/sql/1724236900.sql index 7aa0a7e44e..79879db4d4 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -7,7 +7,9 @@ alter table convoy.configurations add column if not exists cb_success_threshold alter table convoy.configurations add column if not exists cb_observability_window int not null default 5; alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[5, 10]; alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 5; -create index if not exists idx_delivery_attempts_created_at ON convoy.delivery_attempts (created_at); +create index if not exists idx_delivery_attempts_created_at on convoy.delivery_attempts (created_at); +create index if not exists idx_delivery_attempts_event_delivery_id_created_at on convoy.delivery_attempts (event_delivery_id, created_at); +create index if not exists idx_delivery_attempts_event_delivery_id on convoy.delivery_attempts (event_delivery_id); -- +migrate Down alter table convoy.configurations drop column if exists cb_sample_rate; diff --git a/testcon/direct_event_test.go b/testcon/direct_event_test.go index eb00fb7f00..9ef9a5b497 100644 --- a/testcon/direct_event_test.go +++ b/testcon/direct_event_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -10,14 +10,16 @@ import ( "github.com/stretchr/testify/require" ) -func (i *IntegrationTestSuite) Test_DirectEvent_Success_AllSubscriptions() { +func (d *DockerE2EIntegrationTestSuite) Test_DirectEvent_Success_AllSubscriptions() { ctx := context.Background() - t := i.T() + t := d.T() + ownerID := d.DefaultOrg.OwnerID + "_0" + var ports = []int{9909} - c, done := i.initAndStartServers(ports, 2) + c, done := d.initAndStartServers(ports, 2) - endpoint := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID)[0] + endpoint := createEndpoints(t, ctx, c, ports, ownerID)[0] createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"*"}) @@ -29,14 +31,16 @@ func (i *IntegrationTestSuite) Test_DirectEvent_Success_AllSubscriptions() { assertEventCameThrough(t, done, []*convoy.EndpointResponse{endpoint}, []string{traceId, secondTraceId}, []string{}) } -func (i *IntegrationTestSuite) Test_DirectEvent_Success_MustMatchSubscription() { +func (d *DockerE2EIntegrationTestSuite) Test_DirectEvent_Success_MustMatchSubscription() { ctx := context.Background() - t := i.T() + t := d.T() + ownerID := d.DefaultOrg.OwnerID + "_1" + var ports = []int{9910} - c, done := i.initAndStartServers(ports, 1) + c, done := d.initAndStartServers(ports, 1) - endpoint := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID)[0] + endpoint := createEndpoints(t, ctx, c, ports, ownerID)[0] createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"invoice.created"}) diff --git a/testcon/integration_test.go b/testcon/docker_e2e_integration_test.go similarity index 57% rename from testcon/integration_test.go rename to testcon/docker_e2e_integration_test.go index af92e8fc3b..fee94a384a 100644 --- a/testcon/integration_test.go +++ b/testcon/docker_e2e_integration_test.go @@ -1,26 +1,29 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon import ( "context" + "os" + "strings" + "testing" + "time" + "github.com/docker/compose/v2/pkg/api" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" tc "github.com/testcontainers/testcontainers-go/modules/compose" "github.com/testcontainers/testcontainers-go/wait" - "testing" - "time" ) -type IntegrationTestSuite struct { +type DockerE2EIntegrationTestSuite struct { suite.Suite *TestData } -func (i *IntegrationTestSuite) SetupSuite() { - t := i.T() +func (d *DockerE2EIntegrationTestSuite) SetupSuite() { + t := d.T() identifier := tc.StackIdentifier("convoy_docker_test") compose, err := tc.NewDockerComposeWith(tc.WithStackFiles("./testdata/docker-compose-test.yml"), identifier) require.NoError(t, err) @@ -33,22 +36,28 @@ func (i *IntegrationTestSuite) SetupSuite() { t.Cleanup(cancel) // ignore ryuk error - _ = compose.WaitForService("postgres", wait.NewLogStrategy("ready").WithStartupTimeout(60*time.Second)). + err = compose. + WithEnv(map[string]string{ + "CONVOY_LICENSE_KEY": os.Getenv("TEST_LICENSE_KEY"), + }). + WaitForService("postgres", wait.NewLogStrategy("ready").WithStartupTimeout(60*time.Second)). WaitForService("redis_server", wait.NewLogStrategy("Ready to accept connections").WithStartupTimeout(10*time.Second)). WaitForService("migrate", wait.NewLogStrategy("migration up succeeded").WithStartupTimeout(60*time.Second)). Up(ctx, tc.Wait(true), tc.WithRecreate(api.RecreateNever)) - i.TestData = seedTestData(t) -} - -func (i *IntegrationTestSuite) SetupTest() { + if err != nil && !strings.Contains(err.Error(), "Ryuk") && !strings.Contains(err.Error(), "container exited with code 0") { + require.NoError(t, err) + } + d.TestData = seedTestData(t) } -func (i *IntegrationTestSuite) TearDownTest() { +func (d *DockerE2EIntegrationTestSuite) SetupTest() { +} +func (d *DockerE2EIntegrationTestSuite) TearDownTest() { } -func TestIntegrationTestSuite(t *testing.T) { - suite.Run(t, new(IntegrationTestSuite)) +func TestDockerE2EIntegrationTestSuite(t *testing.T) { + suite.Run(t, new(DockerE2EIntegrationTestSuite)) } diff --git a/testcon/integration_test_helper.go b/testcon/docker_e2e_integration_test_helper.go similarity index 87% rename from testcon/integration_test_helper.go rename to testcon/docker_e2e_integration_test_helper.go index 60524f8034..9bdf189c0f 100644 --- a/testcon/integration_test_helper.go +++ b/testcon/docker_e2e_integration_test_helper.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -7,6 +7,16 @@ import ( "context" "errors" "fmt" + "io" + "net" + "net/http" + "os" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + convoy "github.com/frain-dev/convoy-go/v2" "github.com/frain-dev/convoy/api/testdb" "github.com/frain-dev/convoy/auth" @@ -21,36 +31,15 @@ import ( "github.com/frain-dev/convoy/testcon/manifest" "github.com/oklog/ulid/v2" "github.com/stretchr/testify/require" - "io" - "net" - "net/http" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - "time" ) -var once sync.Once -var pDB *postgres.Postgres - -func setEnv(dbPort int, redisPort int) { - _ = os.Setenv("CONVOY_REDIS_HOST", "localhost") - _ = os.Setenv("CONVOY_REDIS_SCHEME", "redis") - _ = os.Setenv("CONVOY_REDIS_PORT", strconv.Itoa(redisPort)) - - _ = os.Setenv("CONVOY_DB_HOST", "localhost") - _ = os.Setenv("CONVOY_DB_SCHEME", "postgres") - _ = os.Setenv("CONVOY_DB_USERNAME", "convoy") - _ = os.Setenv("CONVOY_DB_PASSWORD", "convoy") - _ = os.Setenv("CONVOY_DB_DATABASE", "convoy") - _ = os.Setenv("CONVOY_DB_PORT", strconv.Itoa(dbPort)) -} +var ( + once sync.Once + pDB *postgres.Postgres +) func getConfig() config.Configuration { - err := config.LoadConfig("") + err := config.LoadConfig("./testdata/convoy-host.json") if err != nil { log.Fatal(err) } @@ -73,7 +62,6 @@ type TestData struct { } func seedTestData(t *testing.T) *TestData { - setEnv(5430, 6370) cfg := getConfig() @@ -180,7 +168,6 @@ func startHTTPServer(done chan bool, counter *atomic.Int64, port int) { mux := http.NewServeMux() mux.HandleFunc("/api/convoy", func(w http.ResponseWriter, r *http.Request) { endpoint := "http://" + r.Host + r.URL.Path - fmt.Printf("Received %s request on %s\n", r.Method, endpoint) manifest.IncEndpoint(endpoint) if r.URL.Path != "/api/convoy" { http.NotFound(w, r) @@ -191,6 +178,7 @@ func startHTTPServer(done chan bool, counter *atomic.Int64, port int) { for k, v := range r.URL.Query() { log.Info(fmt.Sprintf("%s: %s\n", k, v)) } + w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("Received a GET request\n")) case "POST": reqBody, err := io.ReadAll(r.Body) @@ -199,7 +187,8 @@ func startHTTPServer(done chan bool, counter *atomic.Int64, port int) { } ev := string(reqBody) - log.Printf("Received: %s\n", reqBody) + fmt.Printf("Received %s request on %s Payload: %s\n", r.Method, endpoint, reqBody) + w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("Received a POST request\n")) manifest.IncEvent(ev) defer func() { @@ -301,14 +290,11 @@ func sendEvent(ctx context.Context, c *convoy.Client, channel string, eUID strin func assertEventCameThrough(t *testing.T, done chan bool, endpoints []*convoy.EndpointResponse, traceIds []string, negativeTraceIds []string) { waitForEvents(t, done) - t.Log("Done waiting. Further wait for 10s") - time.Sleep(10 * time.Second) - manifest.PrintEndpoints() for _, endpoint := range endpoints { hits := manifest.ReadEndpoint(endpoint.TargetUrl) require.NotNil(t, hits) - require.True(t, hits >= 1, endpoint.TargetUrl+" must exist and be non-zero") // ?? + require.Equal(t, hits, len(traceIds), endpoint.TargetUrl+" hits must match events sent") } manifest.PrintEvents() @@ -316,15 +302,13 @@ func assertEventCameThrough(t *testing.T, done chan bool, endpoints []*convoy.En event := fmt.Sprintf(`{"traceId":"%s"}`, traceId) hits := manifest.ReadEvent(event) require.NotNil(t, hits) - require.True(t, hits >= 1, event+" must exist and be non-zero") // ?? + require.Equal(t, hits, len(endpoints), event+" must match number of matched endpoints") } for _, traceId := range negativeTraceIds { event := fmt.Sprintf(`{"traceId":"%s"}`, traceId) hits := manifest.ReadEvent(event) - if !strings.Contains(traceId, "fan-out") { - require.False(t, hits >= 1, event+" must be zero") - } // not sure why fan out ignores sub filter + require.Equal(t, hits, 0, event+" must not exist") } t.Log("Events came through!") @@ -333,7 +317,7 @@ func assertEventCameThrough(t *testing.T, done chan bool, endpoints []*convoy.En func waitForEvents(t *testing.T, done chan bool) { select { case <-done: - case <-time.After(25 * time.Second): + case <-time.After(30 * time.Second): t.Errorf("Time out while waiting for events") } } diff --git a/testcon/fanout_event_test.go b/testcon/fanout_event_test.go index bbf59c2a60..9b05f73726 100644 --- a/testcon/fanout_event_test.go +++ b/testcon/fanout_event_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -11,61 +11,63 @@ import ( "sync/atomic" ) -func (i *IntegrationTestSuite) Test_FanOutEvent_Success_AllSubscriptions() { +func (d *DockerE2EIntegrationTestSuite) Test_FanOutEvent_Success_AllSubscriptions() { ctx := context.Background() - t := i.T() + t := d.T() + ownerId := d.DefaultOrg.OwnerID + "_2" var ports = []int{9911, 9912, 9913} - c, done := i.initAndStartServers(ports, 3*2*2) // 3 endpoints, 2 events each, 2 fan-out operations + c, done := d.initAndStartServers(ports, 3*2) - endpoints := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID) + endpoints := createEndpoints(t, ctx, c, ports, ownerId) traceIds := make([]string, 0) for _, endpoint := range endpoints { createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"*"}) + } - traceId, secondTraceId := "event-fan-out-all-0-"+ulid.Make().String(), "event-fan-out-all-1-"+ulid.Make().String() + traceId, secondTraceId := "event-fan-out-all-0-"+ulid.Make().String(), "event-fan-out-all-1-"+ulid.Make().String() - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "any.event", traceId, i.DefaultOrg.OwnerID)) - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "any.other.event", secondTraceId, i.DefaultOrg.OwnerID)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "any.event", traceId, ownerId)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "any.other.event", secondTraceId, ownerId)) - traceIds = append(traceIds, traceId, secondTraceId) - } + traceIds = append(traceIds, traceId, secondTraceId) assertEventCameThrough(t, done, endpoints, traceIds, []string{}) } -func (i *IntegrationTestSuite) Test_FanOutEvent_Success_MustMatchSubscription() { +func (d *DockerE2EIntegrationTestSuite) Test_FanOutEvent_Success_MustMatchSubscription() { ctx := context.Background() - t := i.T() + t := d.T() + ownerID := d.DefaultOrg.OwnerID + "_3" var ports = []int{9914, 9915, 9916} - c, done := i.initAndStartServers(ports, 3*1) // 3 endpoints, 1 event each + c, done := d.initAndStartServers(ports, 3*1) // 3 endpoints, 1 event each - endpoints := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID) + endpoints := createEndpoints(t, ctx, c, ports, ownerID) traceIds := make([]string, 0) negativeTraceIds := make([]string, 0) for _, endpoint := range endpoints { createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"invoice.fan-out.created"}) + } - traceId, secondTraceId := "event-fan-out-some-0-"+ulid.Make().String(), "event-fan-out-some-1-"+ulid.Make().String() + traceId, secondTraceId := "event-fan-out-some-0-"+ulid.Make().String(), "event-fan-out-some-1-"+ulid.Make().String() - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "mismatched.event.dont.fan.out", traceId, i.DefaultOrg.OwnerID)) - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "invoice.fan-out.created", secondTraceId, i.DefaultOrg.OwnerID)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "mismatched.event.dont.fan.out", traceId, ownerID)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "invoice.fan-out.created", secondTraceId, ownerID)) - traceIds = append(traceIds, secondTraceId) - negativeTraceIds = append(negativeTraceIds, traceId) - } + traceIds = append(traceIds, secondTraceId) + negativeTraceIds = append(negativeTraceIds, traceId) assertEventCameThrough(t, done, endpoints, traceIds, negativeTraceIds) } -func (i *IntegrationTestSuite) initAndStartServers(ports []int, eventCount int64) (*convoy.Client, chan bool) { +func (d *DockerE2EIntegrationTestSuite) initAndStartServers(ports []int, eventCount int64) (*convoy.Client, chan bool) { baseURL := "http://localhost:5015/api/v1" - c := convoy.New(baseURL, i.APIKey, i.DefaultProject.UID) + c := convoy.New(baseURL, d.APIKey, d.DefaultProject.UID) done := make(chan bool, 1) diff --git a/testcon/testdata/convoy-test.json b/testcon/testdata/convoy-docker.json similarity index 100% rename from testcon/testdata/convoy-test.json rename to testcon/testdata/convoy-docker.json diff --git a/testcon/testdata/convoy-host.json b/testcon/testdata/convoy-host.json new file mode 100644 index 0000000000..20cb484e30 --- /dev/null +++ b/testcon/testdata/convoy-host.json @@ -0,0 +1,56 @@ +{ + "host": "localhost:5015", + "database": { + "host": "localhost", + "username": "convoy", + "password": "convoy", + "database": "convoy", + "port": 5430 + }, + "redis": { + "port": 6370, + "host": "localhost" + }, + "metrics": { + "metrics_backend": "prometheus", + "prometheus_metrics": { + "sample_time": 10 + } + }, + "instance_ingest_rate": 50, + "api_rate_limit_enabled": false, + "auth": { + "jwt": { + "enabled": true + }, + "native": { + "enabled": true + }, + "file": { + "basic": [ + { + "username": "test", + "password": "test", + "role": { + "type": "super_user" + } + }, + { + "username": "default@user.com", + "password": "password", + "role": { + "type": "super_user" + } + }, + { + "username": "test-group-filter", + "password": "test-group-filter", + "role": { + "group": "abcdef", + "type": "super_user" + } + } + ] + } + } +} diff --git a/testcon/testdata/docker-compose-test.yml b/testcon/testdata/docker-compose-test.yml index 0798d55a90..d2b80d0a1a 100644 --- a/testcon/testdata/docker-compose-test.yml +++ b/testcon/testdata/docker-compose-test.yml @@ -10,8 +10,10 @@ services: context: ../../ dockerfile: Dockerfile.dev command: [ "/start.sh" ] + environment: + - CONVOY_LICENSE_KEY volumes: - - ./convoy-test.json:/convoy.json + - ./convoy-docker.json:/convoy.json restart: on-failure ports: - "5015:5005" @@ -26,7 +28,7 @@ services: dockerfile: Dockerfile.dev entrypoint: ["./cmd", "migrate", "up"] volumes: - - ./convoy-test.json:/convoy.json + - ./convoy-docker.json:/convoy.json restart: on-failure depends_on: postgres: @@ -37,8 +39,10 @@ services: context: ../../ dockerfile: Dockerfile.dev entrypoint: ["./cmd", "agent", "--config", "convoy.json"] + environment: + - CONVOY_LICENSE_KEY volumes: - - ./convoy-test.json:/convoy.json + - ./convoy-docker.json:/convoy.json restart: on-failure ports: - "5018:5008" diff --git a/web/ui/dashboard/src/app/components/tag/tag.component.ts b/web/ui/dashboard/src/app/components/tag/tag.component.ts index 7249bd83bc..a6c85d9248 100644 --- a/web/ui/dashboard/src/app/components/tag/tag.component.ts +++ b/web/ui/dashboard/src/app/components/tag/tag.component.ts @@ -9,11 +9,10 @@ import { STATUS_COLOR } from 'src/app/models/global.model'; template: ` `, - host: { class: 'rounded-22px w-fit text-center text-12 justify-between gap-x-4px disabled:opacity-50', '[class]': 'classes' } + host: { class: 'rounded-22px w-fit text-center text-12 justify-between gap-x-4px disabled:opacity-50 flex items-center justify-center', '[class]': 'classes' } }) export class TagComponent implements OnInit { @Input('className') class!: string; - @Input('fill') fill: 'outline' | 'soft' | 'solid' | 'soft-outline' = 'soft'; @Input('color') color: 'primary' | 'error' | 'success' | 'warning' | 'neutral' = 'neutral'; @Input('size') size: 'sm' | 'md' | 'lg' = 'md'; diff --git a/web/ui/dashboard/src/app/portal/portal.component.html b/web/ui/dashboard/src/app/portal/portal.component.html index 539ebb87ca..4fcabc375e 100644 --- a/web/ui/dashboard/src/app/portal/portal.component.html +++ b/web/ui/dashboard/src/app/portal/portal.component.html @@ -1,23 +1,31 @@ -
- +
+ + + +
+ + convoy logo +
- diff --git a/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html b/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html index 1adda9d2d5..b84c0b3714 100644 --- a/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html +++ b/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html @@ -81,7 +81,15 @@
-

Alert Configuration

+
+

Alert Configuration

+
+ + + + Business +
+
-
+
@@ -160,10 +168,18 @@
-

- Endpoint Timeout - How many seconds should Convoy wait for a response from this endpoint before timing out? -

+
+

+ Endpoint Timeout + How many seconds should Convoy wait for a response from this endpoint before timing out? +

+
+ + + + Business +
+
-
-

- Search Period - This will trigger search re-tokenization and only events within this period will be available for search. -

+
+
+

+ Search Period + This will trigger search re-tokenization and only events within this period will be available for search. +

+
+ + + + Business +
+
- -
- -
hour(s)
-
- Enter search policy value - + + +
+ +
hour(s)
+
+ Enter search policy value + +
diff --git a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts index c2179653a0..b043f7feea 100644 --- a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts +++ b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts @@ -1,11 +1,12 @@ -import {Component, ElementRef, EventEmitter, inject, Input, OnInit, Output, ViewChild} from '@angular/core'; -import {FormArray, FormBuilder, FormGroup, Validators} from '@angular/forms'; -import {ActivatedRoute, Router} from '@angular/router'; -import {PROJECT, VERSIONS} from 'src/app/models/project.model'; -import {GeneralService} from 'src/app/services/general/general.service'; -import {PrivateService} from '../../private.service'; -import {CreateProjectComponentService} from './create-project-component.service'; -import {RbacService} from 'src/app/services/rbac/rbac.service'; +import { Component, ElementRef, EventEmitter, inject, Input, OnInit, Output, ViewChild } from '@angular/core'; +import { FormArray, FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { ActivatedRoute, Router } from '@angular/router'; +import { PROJECT, VERSIONS } from 'src/app/models/project.model'; +import { GeneralService } from 'src/app/services/general/general.service'; +import { PrivateService } from '../../private.service'; +import { CreateProjectComponentService } from './create-project-component.service'; +import { RbacService } from 'src/app/services/rbac/rbac.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; interface TAB { label: string; @@ -95,7 +96,15 @@ export class CreateProjectComponent implements OnInit { activeTab = this.tabs[0]; events = ['endpoint.created', 'endpoint.deleted', 'endpoint.updated', 'eventdelivery.success', 'eventdelivery.failed', 'project.updated']; - constructor(private formBuilder: FormBuilder, private createProjectService: CreateProjectComponentService, private generalService: GeneralService, private privateService: PrivateService, public router: Router, private route: ActivatedRoute) {} + constructor( + private formBuilder: FormBuilder, + private createProjectService: CreateProjectComponentService, + private generalService: GeneralService, + private privateService: PrivateService, + public router: Router, + private route: ActivatedRoute, + public licenseService: LicensesService + ) {} async ngOnInit() { if (this.action === 'update') this.getProjectDetails(); @@ -183,8 +192,6 @@ export class CreateProjectComponent implements OnInit { } const projectData = this.getProjectData(); - console.log(projectData); - this.isCreatingProject = true; try { @@ -201,6 +208,7 @@ export class CreateProjectComponent implements OnInit { this.projectDetails = response.data.project; if (projectFormModal) projectFormModal.style.overflowY = 'hidden'; this.tokenDialog.nativeElement.showModal(); + this.licenseService.setLicenses(); } catch (error) { this.isCreatingProject = false; } diff --git a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts index 1792d827b5..91000b9ca9 100644 --- a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts +++ b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts @@ -16,6 +16,7 @@ import { TokenModalComponent } from '../token-modal/token-modal.component'; import { PermissionDirective } from '../permission/permission.directive'; import { NotificationComponent } from 'src/app/components/notification/notification.component'; import { ConfigButtonComponent } from '../config-button/config-button.component'; +import { TagComponent } from 'src/app/components/tag/tag.component'; @NgModule({ declarations: [CreateProjectComponent], @@ -46,7 +47,8 @@ import { ConfigButtonComponent } from '../config-button/config-button.component' PermissionDirective, DialogDirective, NotificationComponent, - ConfigButtonComponent + ConfigButtonComponent, + TagComponent ], exports: [CreateProjectComponent] }) diff --git a/web/ui/dashboard/src/app/private/components/create-source/create-source.component.html b/web/ui/dashboard/src/app/private/components/create-source/create-source.component.html index cd30ac52d9..efd98db800 100644 --- a/web/ui/dashboard/src/app/private/components/create-source/create-source.component.html +++ b/web/ui/dashboard/src/app/private/components/create-source/create-source.component.html @@ -286,13 +286,13 @@

Configure SQS

-
-

Configure Kafka

- - doc icon - Docs - -
+
+

Configure Kafka

+ + doc icon + Docs + +
Please add at least one broker address @@ -412,11 +412,19 @@

Custom response

-

Transform

+
+

Transform

+
+ + + + Business +
+

Transform request body of events with a javascript function.

- +
diff --git a/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts b/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts index 7c7cd3e8bb..093049b542 100644 --- a/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts +++ b/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts @@ -6,6 +6,7 @@ import { GeneralService } from 'src/app/services/general/general.service'; import { PrivateService } from '../../private.service'; import { CreateSourceService } from './create-source.service'; import { RbacService } from 'src/app/services/rbac/rbac.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-create-source', @@ -175,7 +176,7 @@ export class CreateSourceComponent implements OnInit { sourceURL!: string; showTransformDialog = false; - constructor(private formBuilder: FormBuilder, private createSourceService: CreateSourceService, public privateService: PrivateService, private route: ActivatedRoute, private router: Router, private generalService: GeneralService) {} + constructor(private formBuilder: FormBuilder, private createSourceService: CreateSourceService, public privateService: PrivateService, private route: ActivatedRoute, private router: Router, private generalService: GeneralService, public licenseService: LicensesService) {} async ngOnInit() { if (this.privateService.getProjectDetails.type === 'incoming') diff --git a/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts b/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts index 71990fc8f7..7b22aee15a 100644 --- a/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts +++ b/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts @@ -19,6 +19,7 @@ import { NotificationComponent } from 'src/app/components/notification/notificat import { ConfigButtonComponent } from '../config-button/config-button.component'; import { SourceURLComponent } from './source-url/source-url.component'; import { CreateTransformFunctionComponent } from '../create-transform-function/create-transform-function.component'; +import { TagComponent } from 'src/app/components/tag/tag.component'; @NgModule({ declarations: [CreateSourceComponent], @@ -45,7 +46,8 @@ import { CreateTransformFunctionComponent } from '../create-transform-function/c NotificationComponent, ConfigButtonComponent, SourceURLComponent, - CreateTransformFunctionComponent + CreateTransformFunctionComponent, + TagComponent ], exports: [CreateSourceComponent] }) diff --git a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html index be5c241fae..93885e912e 100644 --- a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html +++ b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html @@ -144,11 +144,19 @@

Webhook Subscription Configuration

-

Events Filter

+
+

Events Filter

+
+ + + + Business +
+

Filter events received by request body and header.

- +
@@ -164,11 +172,21 @@

Events Filter

-

Transform

+
+
+

Transform

+
+
+ + + + Business +
+

Transform request body of events with a javascript function.

- +
diff --git a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts index c613b1d4ad..f19972d426 100644 --- a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts +++ b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts @@ -9,6 +9,7 @@ import { CreateSourceComponent } from '../create-source/create-source.component' import { CreateSubscriptionService } from './create-subscription.service'; import { RbacService } from 'src/app/services/rbac/rbac.service'; import { SUBSCRIPTION } from 'src/app/models/subscription'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-create-subscription', @@ -78,7 +79,7 @@ export class CreateSubscriptionComponent implements OnInit { subscription!: SUBSCRIPTION; currentRoute = window.location.pathname.split('/').reverse()[0]; - constructor(private formBuilder: FormBuilder, private privateService: PrivateService, private createSubscriptionService: CreateSubscriptionService, private route: ActivatedRoute, private router: Router) {} + constructor(private formBuilder: FormBuilder, private privateService: PrivateService, private createSubscriptionService: CreateSubscriptionService, private route: ActivatedRoute, private router: Router, public licenseService: LicensesService) {} async ngOnInit() { this.isLoadingForm = true; diff --git a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts index d60ce86957..c32b47a2dd 100644 --- a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts +++ b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts @@ -22,6 +22,7 @@ import { NotificationComponent } from 'src/app/components/notification/notificat import { CreateTransformFunctionComponent } from '../create-transform-function/create-transform-function.component'; import { ConfigButtonComponent } from '../config-button/config-button.component'; import { SourceURLComponent } from '../create-source/source-url/source-url.component'; +import { TagComponent } from 'src/app/components/tag/tag.component'; @NgModule({ declarations: [CreateSubscriptionComponent], @@ -51,7 +52,8 @@ import { SourceURLComponent } from '../create-source/source-url/source-url.compo NotificationComponent, CreateTransformFunctionComponent, ConfigButtonComponent, - SourceURLComponent + SourceURLComponent, + TagComponent ], exports: [CreateSubscriptionComponent] }) diff --git a/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html b/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html index 838c51f78f..19f377cb3b 100644 --- a/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html +++ b/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html @@ -1,7 +1,7 @@
-
+
search icon @@ -21,17 +21,7 @@
Date
- {{ queryParams.startDate | date: 'dd/MM/yy, h:mm a' }} - {{ queryParams.endDate | date: 'dd/MM/yy, h:mm a' }} + {{ queryParams.startDate | date : 'dd/MM/yy, h:mm a' }} - {{ queryParams.endDate | date : 'dd/MM/yy, h:mm a' }} +
+ +
+
-
- -
- -
- - -
-
-

Portal Links

- -
+ +
+
-
-
- - + +
+
+

Portal Links

+ +
+
- - -
- - + Endpoints + + +
+ + +
-
-
-
- - - - - Create Portal Link - +
+
+ + + + + Create Portal Link + -
-
-
-
-
{{ link.name }}
-
- {{ link.endpoints_metadata.length }} Endpoint{{ link.endpoints_metadata.length > 1 ? 's' : '' }} +
+
+
+
+
{{ link.name }}
+
+ {{ link.endpoints_metadata.length }} Endpoint{{ link.endpoints_metadata.length > 1 ? 's' : '' }} +
-
-
- +
+ -
    -
  • - -
    • -
  • - -
    • -
+
    +
  • + +
    • +
  • + +
    • +
+
-
-
- -
- {{ link.url }} - - - - - - -
- -
- + +
-
- + - - + + +
+ +
+
diff --git a/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts b/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts index 828fca4bcc..bd20cba085 100644 --- a/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts +++ b/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts @@ -24,6 +24,7 @@ import { PermissionDirective } from 'src/app/private/components/permission/permi import { LoaderModule } from 'src/app/private/components/loader/loader.module'; import { EndpointFilterComponent } from 'src/app/private/components/endpoints-filter/endpoints-filter.component'; import { TagComponent } from 'src/app/components/tag/tag.component'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-portal-links', @@ -70,10 +71,10 @@ export class PortalLinksComponent implements OnInit { @ViewChild('linksEndpointFilter', { static: true }) linksEndpointFilter!: ElementRef; linksEndpointFilter$!: Observable; - constructor(public privateService: PrivateService, public router: Router, private portalLinksService: PortalLinksService, private route: ActivatedRoute, private generalService: GeneralService) {} + constructor(public privateService: PrivateService, public router: Router, private portalLinksService: PortalLinksService, private route: ActivatedRoute, private generalService: GeneralService, public licenseService: LicensesService) {} ngOnInit() { - this.getPortalLinks(); + if(this.licenseService.hasLicense('PORTAL_LINKS')) this.getPortalLinks(); const urlParam = this.route.snapshot.params.id; if (urlParam) { diff --git a/web/ui/dashboard/src/app/private/pages/project/project.component.html b/web/ui/dashboard/src/app/private/pages/project/project.component.html index a0ea920165..59a1d2109b 100644 --- a/web/ui/dashboard/src/app/private/pages/project/project.component.html +++ b/web/ui/dashboard/src/app/private/pages/project/project.component.html @@ -27,7 +27,15 @@
    -
  • - +
    + + + + Business +
diff --git a/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts b/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts index bf3444b8a5..2f0bf11056 100644 --- a/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts +++ b/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts @@ -1,6 +1,7 @@ -import {Location} from '@angular/common'; -import {Component, OnInit} from '@angular/core'; -import {ActivatedRoute, Router} from '@angular/router'; +import { Location } from '@angular/common'; +import { Component, OnInit } from '@angular/core'; +import { ActivatedRoute, Router } from '@angular/router'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; export type SETTINGS = 'organisation settings' | 'configuration settings' | 'personal access tokens' | 'team'; @@ -13,14 +14,14 @@ export class SettingsComponent implements OnInit { activePage: SETTINGS = 'organisation settings'; settingsMenu: { name: SETTINGS; icon: string; svg: 'stroke' | 'fill' }[] = [ { name: 'organisation settings', icon: 'org', svg: 'fill' }, - { name: 'team', icon: 'team', svg: 'stroke' }, + { name: 'team', icon: 'team', svg: 'stroke' } // { name: 'configuration settings', icon: 'settings', svg: 'fill' } ]; - constructor(private router: Router, private route: ActivatedRoute, private location: Location) {} + constructor(private router: Router, private route: ActivatedRoute, public licenseService: LicensesService) {} ngOnInit() { - this.toggleActivePage(this.route.snapshot.queryParams?.activePage ?? 'organisation settings'); + if (this.licenseService.hasLicense('CREATE_ORG_MEMBER')) this.toggleActivePage(this.route.snapshot.queryParams?.activePage ?? 'organisation settings'); } toggleActivePage(activePage: SETTINGS) { diff --git a/web/ui/dashboard/src/app/private/private.component.html b/web/ui/dashboard/src/app/private/private.component.html index 024c2037df..ed44f8f909 100644 --- a/web/ui/dashboard/src/app/private/private.component.html +++ b/web/ui/dashboard/src/app/private/private.component.html @@ -46,11 +46,17 @@ -
  • +
  • +
    + + + + Business +
    • diff --git a/web/ui/dashboard/src/app/private/private.component.ts b/web/ui/dashboard/src/app/private/private.component.ts index 00862ebfaf..3602394a6d 100644 --- a/web/ui/dashboard/src/app/private/private.component.ts +++ b/web/ui/dashboard/src/app/private/private.component.ts @@ -7,6 +7,7 @@ import { FormBuilder, FormControl, FormGroup, Validators } from '@angular/forms' import { JwtHelperService } from '@auth0/angular-jwt'; import { differenceInSeconds } from 'date-fns'; import { Observable, Subscription } from 'rxjs'; +import { LicensesService } from '../services/licenses/licenses.service'; @Component({ selector: 'app-private', @@ -47,13 +48,13 @@ export class PrivateComponent implements OnInit { private jwtHelper: JwtHelperService = new JwtHelperService(); private shouldShowOrgSubscription: Subscription | undefined; - constructor(private generalService: GeneralService, public router: Router, public privateService: PrivateService, private formBuilder: FormBuilder) {} + constructor(private generalService: GeneralService, public router: Router, public privateService: PrivateService, private formBuilder: FormBuilder, public licenseService: LicensesService) {} async ngOnInit() { this.shouldShowOrgModal(); this.checkIfTokenIsExpired(); - await Promise.all([this.getConfiguration(), this.getUserDetails(), this.getOrganizations()]); + await Promise.all([this.getConfiguration(), this.licenseService.setLicenses(), this.getUserDetails(), this.getOrganizations()]); } ngOnDestroy() { @@ -71,9 +72,9 @@ export class PrivateComponent implements OnInit { return authDetails ? JSON.parse(authDetails) : false; } - shouldMountAppRouter(): boolean { - return !this.isLoadingOrganisations && (Boolean(this.organisations?.length) || this.router.url.startsWith('/user-settings')) - } + shouldMountAppRouter(): boolean { + return !this.isLoadingOrganisations && (Boolean(this.organisations?.length) || this.router.url.startsWith('/user-settings')); + } async getConfiguration() { try { @@ -162,6 +163,7 @@ export class PrivateComponent implements OnInit { this.generalService.showNotification({ style: 'success', message: response.message }); this.creatingOrganisation = false; this.dialog.nativeElement.close(); + this.licenseService.setLicenses(); await this.getOrganizations(true); this.selectOrganisation(response.data); diff --git a/web/ui/dashboard/src/app/public/login/login.component.html b/web/ui/dashboard/src/app/public/login/login.component.html index d9333a1a50..191952687f 100644 --- a/web/ui/dashboard/src/app/public/login/login.component.html +++ b/web/ui/dashboard/src/app/public/login/login.component.html @@ -30,7 +30,7 @@ - +
    diff --git a/web/ui/dashboard/src/app/public/login/login.component.ts b/web/ui/dashboard/src/app/public/login/login.component.ts index 07f0c97f2c..c31a83ee7a 100644 --- a/web/ui/dashboard/src/app/public/login/login.component.ts +++ b/web/ui/dashboard/src/app/public/login/login.component.ts @@ -9,6 +9,7 @@ import { LoaderModule } from 'src/app/private/components/loader/loader.module'; import { PrivateService } from 'src/app/private/private.service'; import { ORGANIZATION_DATA } from 'src/app/models/organisation.model'; import { SignupService } from '../signup/signup.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'app-login', @@ -29,10 +30,11 @@ export class LoginComponent implements OnInit { isSignupEnabled = false; organisations?: ORGANIZATION_DATA[]; - constructor(private formBuilder: FormBuilder, public router: Router, private loginService: LoginService, private signupService: SignupService, private privateService: PrivateService) {} + constructor(private formBuilder: FormBuilder, public router: Router, private loginService: LoginService, private signupService: SignupService, private privateService: PrivateService, public licenseService: LicensesService) {} ngOnInit() { this.getSignUpConfig(); + this.licenseService.setLicenses(); } async getSignUpConfig() { diff --git a/web/ui/dashboard/src/app/public/signup/signup.component.ts b/web/ui/dashboard/src/app/public/signup/signup.component.ts index 179ca11048..d0e1c708ce 100644 --- a/web/ui/dashboard/src/app/public/signup/signup.component.ts +++ b/web/ui/dashboard/src/app/public/signup/signup.component.ts @@ -7,6 +7,7 @@ import { InputDirective, InputErrorComponent, InputFieldDirective, LabelComponen import { LoaderModule } from 'src/app/private/components/loader/loader.module'; import { HubspotService } from 'src/app/services/hubspot/hubspot.service'; import { SignupService } from './signup.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-signup', @@ -27,9 +28,13 @@ export class SignupComponent implements OnInit { org_name: ['', Validators.required] }); - constructor(private formBuilder: FormBuilder, private signupService: SignupService, public router: Router, private hubspotService: HubspotService) {} + constructor(private formBuilder: FormBuilder, private signupService: SignupService, public router: Router, private hubspotService: HubspotService, private licenseService: LicensesService) {} - ngOnInit(): void {} + ngOnInit(): void { + this.licenseService.setLicenses(); + + if (!this.licenseService.hasLicense('CREATE_USER')) this.router.navigateByUrl('/login'); + } async signup() { if (this.signupForm.invalid) return this.signupForm.markAllAsTouched(); diff --git a/web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts b/web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts new file mode 100644 index 0000000000..13b7108200 --- /dev/null +++ b/web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { LicensesService } from './licenses.service'; + +describe('LicensesService', () => { + let service: LicensesService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(LicensesService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/web/ui/dashboard/src/app/services/licenses/licenses.service.ts b/web/ui/dashboard/src/app/services/licenses/licenses.service.ts new file mode 100644 index 0000000000..0ed3a106a7 --- /dev/null +++ b/web/ui/dashboard/src/app/services/licenses/licenses.service.ts @@ -0,0 +1,51 @@ +import { Injectable } from '@angular/core'; +import { HttpService } from '../http/http.service'; +import { HTTP_RESPONSE } from 'src/app/models/global.model'; + +@Injectable({ + providedIn: 'root' +}) +export class LicensesService { + constructor(private http: HttpService) {} + + getLicenses(): Promise { + return new Promise(async (resolve, reject) => { + try { + const response = await this.http.request({ + url: `/license/features`, + method: 'get' + }); + + return resolve(response); + } catch (error) { + return reject(error); + } + }); + } + + + async setLicenses() { + try { + const response = await this.getLicenses(); + + let allowedLicenses: any[] = []; + Object.entries(response.data).forEach(([key, entry]: any) => { + if (entry.allowed) allowedLicenses.push(key); + }); + + localStorage.setItem('licenses', JSON.stringify(allowedLicenses)); + } catch {} + } + + hasLicense(license: string) { + const savedLicenses = localStorage.getItem('licenses'); + if (savedLicenses) { + const licenses = JSON.parse(savedLicenses); + const userHasLicense = licenses.includes(license); + + return userHasLicense; + } + + return false; + } +} diff --git a/web/ui/dashboard/src/assets/img/svg/page-locked.svg b/web/ui/dashboard/src/assets/img/svg/page-locked.svg new file mode 100644 index 0000000000..5a8679199c --- /dev/null +++ b/web/ui/dashboard/src/assets/img/svg/page-locked.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/web/ui/dashboard/src/index.html b/web/ui/dashboard/src/index.html index 26de11e99f..cd9cbe39b8 100644 --- a/web/ui/dashboard/src/index.html +++ b/web/ui/dashboard/src/index.html @@ -149,6 +149,12 @@ /> + + + + diff --git a/worker/task/process_broadcast_event_creation.go b/worker/task/process_broadcast_event_creation.go index 6fbb699633..8a9dcdea62 100644 --- a/worker/task/process_broadcast_event_creation.go +++ b/worker/task/process_broadcast_event_creation.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" - "gopkg.in/guregu/null.v4" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/frain-dev/convoy/queue/redis" "github.com/frain-dev/convoy" @@ -28,7 +30,7 @@ var ( defaultBroadcastDelay = 30 * time.Second ) -func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, subscriptionsTable memorystore.ITable) func(context.Context, *asynq.Task) error { +func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, licenser license.Licenser, subscriptionsTable memorystore.ITable) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) (err error) { var broadcastEvent models.BroadcastEvent @@ -77,7 +79,7 @@ func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, ev AcknowledgedAt: null.TimeFrom(time.Now()), } - subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, subscriptions, true) + subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, licenser, subscriptions, true) if err != nil { return &EndpointError{Err: fmt.Errorf("failed to match subscriptions using filter, err: %s", err.Error()), delay: defaultBroadcastDelay} } @@ -103,7 +105,7 @@ func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, ev return nil } - err = writeEventDeliveriesToQueue(ctx, ss, event, project, eventDeliveryRepo, eventQueue, deviceRepo, endpointRepo) + err = writeEventDeliveriesToQueue(ctx, ss, event, project, eventDeliveryRepo, eventQueue, deviceRepo, endpointRepo, licenser) if err != nil { log.WithError(err).Error(ErrFailedToWriteToQueue) return &EndpointError{Err: fmt.Errorf("%s, err: %s", ErrFailedToWriteToQueue.Error(), err.Error()), delay: defaultBroadcastDelay} diff --git a/worker/task/process_broadcast_event_creation_test.go b/worker/task/process_broadcast_event_creation_test.go index 2f5b801b52..2e9c7ca77b 100644 --- a/worker/task/process_broadcast_event_creation_test.go +++ b/worker/task/process_broadcast_event_creation_test.go @@ -146,7 +146,7 @@ func TestProcessBroadcastEventCreation(t *testing.T) { fn := ProcessBroadcastEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, - args.deviceRepo, args.subTable) + args.deviceRepo, args.licenser, args.subTable) err = fn(context.Background(), task) if tt.wantErr { require.NotNil(t, err) diff --git a/worker/task/process_dynamic_event_creation.go b/worker/task/process_dynamic_event_creation.go index 7e80c2d8b3..d1ebdf4cb1 100644 --- a/worker/task/process_dynamic_event_creation.go +++ b/worker/task/process_dynamic_event_creation.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" - "gopkg.in/guregu/null.v4" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/frain-dev/convoy/pkg/msgpack" "github.com/google/uuid" @@ -23,7 +25,7 @@ import ( "github.com/oklog/ulid/v2" ) -func ProcessDynamicEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository) func(context.Context, *asynq.Task) error { +func ProcessDynamicEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, licenser license.Licenser) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var dynamicEvent models.DynamicEvent @@ -85,7 +87,7 @@ func ProcessDynamicEventCreation(endpointRepo datastore.EndpointRepository, even return writeEventDeliveriesToQueue( ctx, []datastore.Subscription{*s}, event, project, eventDeliveryRepo, - eventQueue, deviceRepo, endpointRepo, + eventQueue, deviceRepo, endpointRepo, licenser, ) } } diff --git a/worker/task/process_dynamic_event_creation_test.go b/worker/task/process_dynamic_event_creation_test.go index 193b5b0585..57447e8028 100644 --- a/worker/task/process_dynamic_event_creation_test.go +++ b/worker/task/process_dynamic_event_creation_test.go @@ -190,7 +190,7 @@ func TestProcessDynamicEventCreation(t *testing.T) { task := asynq.NewTask(string(convoy.EventProcessor), job.Payload, asynq.Queue(string(convoy.EventQueue)), asynq.ProcessIn(job.Delay)) - fn := ProcessDynamicEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo) + fn := ProcessDynamicEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo, args.licenser) err = fn(context.Background(), task) if tt.wantErr { require.NotNil(t, err) diff --git a/worker/task/process_event_creation.go b/worker/task/process_event_creation.go index a6bf845439..6d99b155a1 100644 --- a/worker/task/process_event_creation.go +++ b/worker/task/process_event_creation.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" - "gopkg.in/guregu/null.v4" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/frain-dev/convoy/pkg/flatten" "github.com/frain-dev/convoy" @@ -47,7 +49,7 @@ type CreateEvent struct { func ProcessEventCreation( endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, - subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, + subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, licenser license.Licenser, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var createEvent CreateEvent @@ -82,7 +84,7 @@ func ProcessEventCreation( event = createEvent.Event } - subscriptions, err := findSubscriptions(ctx, endpointRepo, subRepo, project, event, createEvent.CreateSubscription) + subscriptions, err := findSubscriptions(ctx, endpointRepo, subRepo, licenser, project, event, createEvent.CreateSubscription) if err != nil { return &EndpointError{Err: err, delay: defaultDelay} } @@ -112,12 +114,12 @@ func ProcessEventCreation( return writeEventDeliveriesToQueue( ctx, subscriptions, event, project, eventDeliveryRepo, - eventQueue, deviceRepo, endpointRepo, + eventQueue, deviceRepo, endpointRepo, licenser, ) } } -func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore.Subscription, event *datastore.Event, project *datastore.Project, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, deviceRepo datastore.DeviceRepository, endpointRepo datastore.EndpointRepository) error { +func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore.Subscription, event *datastore.Event, project *datastore.Project, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, deviceRepo datastore.DeviceRepository, endpointRepo datastore.EndpointRepository, licenser license.Licenser) error { ec := &EventDeliveryConfig{project: project} eventDeliveries := make([]*datastore.EventDelivery, 0) @@ -152,7 +154,7 @@ func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore. raw := event.Raw data := event.Data - if s.Function.Ptr() != nil && !util.IsStringEmpty(s.Function.String) { + if s.Function.Ptr() != nil && !util.IsStringEmpty(s.Function.String) && licenser.Transformations() { var payload map[string]interface{} err = json.Unmarshal(event.Data, &payload) if err != nil { @@ -252,7 +254,7 @@ func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore. } func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepository, - subRepo datastore.SubscriptionRepository, project *datastore.Project, event *datastore.Event, shouldCreateSubscription bool, + subRepo datastore.SubscriptionRepository, licenser license.Licenser, project *datastore.Project, event *datastore.Event, shouldCreateSubscription bool, ) ([]datastore.Subscription, error) { var subscriptions []datastore.Subscription var err error @@ -284,7 +286,7 @@ func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepos subs = matchSubscriptions(string(event.EventType), subs) - subs, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, subs, false) + subs, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, licenser, subs, false) if err != nil { return subscriptions, &EndpointError{Err: errors.New("error fetching subscriptions for event type"), delay: defaultDelay} } @@ -297,7 +299,7 @@ func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepos return nil, &EndpointError{Err: err, delay: defaultDelay} } - subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, subscriptions, false) + subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, licenser, subscriptions, false) if err != nil { log.WithError(err).Error("error find a matching subscription for this source") return subscriptions, &EndpointError{Err: errors.New("error find a matching subscription for this source"), delay: defaultDelay} @@ -307,7 +309,11 @@ func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepos return subscriptions, nil } -func matchSubscriptionsUsingFilter(ctx context.Context, e *datastore.Event, subRepo datastore.SubscriptionRepository, subscriptions []datastore.Subscription, soft bool) ([]datastore.Subscription, error) { +func matchSubscriptionsUsingFilter(ctx context.Context, e *datastore.Event, subRepo datastore.SubscriptionRepository, licenser license.Licenser, subscriptions []datastore.Subscription, soft bool) ([]datastore.Subscription, error) { + if !licenser.AdvancedSubscriptions() { + return subscriptions, nil + } + var matched []datastore.Subscription // payload is interface{} and not map[string]interface{} because diff --git a/worker/task/process_event_creation_test.go b/worker/task/process_event_creation_test.go index 5cb7480e68..f1a8931005 100644 --- a/worker/task/process_event_creation_test.go +++ b/worker/task/process_event_creation_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -31,6 +33,7 @@ type args struct { subRepo datastore.SubscriptionRepository deviceRepo datastore.DeviceRepository subTable memorystore.ITable + licenser license.Licenser } func provideArgs(ctrl *gomock.Controller) *args { @@ -56,6 +59,7 @@ func provideArgs(ctrl *gomock.Controller) *args { eventQueue: mockQueuer, subRepo: subRepo, subTable: subTable, + licenser: mocks.NewMockLicenser(ctrl), } } @@ -137,6 +141,86 @@ func TestProcessEventCreated(t *testing.T) { ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + + q, _ := args.eventQueue.(*mocks.MockQueuer) + q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(1).Return(nil) + }, + wantErr: false, + }, + + { + name: "should_skip_filter_comparing_for_license_check_failed", + event: &CreateEvent{ + Params: CreateEventTaskParams{ + UID: ulid.Make().String(), + EventType: "*", + SourceID: "source-id-1", + ProjectID: "project-id-1", + EndpointID: "endpoint-id-1", + Data: []byte(`{}`), + }, + }, + dbFn: func(args *args) { + project := &datastore.Project{ + UID: "project-id-1", + Type: datastore.OutgoingProject, + Config: &datastore.ProjectConfig{ + Strategy: &datastore.StrategyConfiguration{ + Type: datastore.LinearStrategyProvider, + Duration: 10, + RetryCount: 3, + }, + }, + } + + g, _ := args.projectRepo.(*mocks.MockProjectRepository) + g.EXPECT().FetchProjectByID(gomock.Any(), "project-id-1").Times(1).Return( + project, + nil, + ) + + a, _ := args.endpointRepo.(*mocks.MockEndpointRepository) + + endpoint := &datastore.Endpoint{UID: "endpoint-id-1"} + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()).Times(1).Return(endpoint, nil) + + s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) + subscriptions := []datastore.Subscription{ + { + UID: "456", + EndpointID: "endpoint-id-1", + Type: datastore.SubscriptionTypeAPI, + FilterConfig: &datastore.FilterConfiguration{ + EventTypes: []string{"*"}, + Filter: datastore.FilterSchema{ + Headers: nil, + Body: map[string]interface{}{"key": "value"}, + }, + }, + }, + } + + endpoint = &datastore.Endpoint{UID: "endpoint-id-1"} + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()).Times(1).Return(endpoint, nil) + + s.EXPECT().FindSubscriptionsByEndpointID(gomock.Any(), "project-id-1", "endpoint-id-1").Times(1).Return(subscriptions, nil) + + e, _ := args.eventRepo.(*mocks.MockEventRepository) + e.EXPECT().FindEventByID(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil, datastore.ErrEventNotFound) + e.EXPECT().CreateEvent(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + endpoint = &datastore.Endpoint{UID: "098", Url: "https://google.com", Status: datastore.ActiveEndpointStatus} + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()). + Times(1).Return(endpoint, nil) + + ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) + ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(false) + q, _ := args.eventQueue.(*mocks.MockQueuer) q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(1).Return(nil) }, @@ -284,6 +368,9 @@ func TestProcessEventCreated(t *testing.T) { ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + q, _ := args.eventQueue.(*mocks.MockQueuer) q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(2).Return(nil) }, @@ -365,6 +452,9 @@ func TestProcessEventCreated(t *testing.T) { }, nil, ) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + e, _ := args.eventRepo.(*mocks.MockEventRepository) e.EXPECT().FindEventByID(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil, datastore.ErrEventNotFound) e.EXPECT().CreateEvent(gomock.Any(), gomock.Any()).Times(1).Return(nil) @@ -442,6 +532,9 @@ func TestProcessEventCreated(t *testing.T) { ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + q, _ := args.eventQueue.(*mocks.MockQueuer) q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(1).Return(nil) }, @@ -468,7 +561,7 @@ func TestProcessEventCreated(t *testing.T) { task := asynq.NewTask(string(convoy.EventProcessor), job.Payload, asynq.Queue(string(convoy.EventQueue)), asynq.ProcessIn(job.Delay)) - fn := ProcessEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo) + fn := ProcessEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo, args.licenser) err = fn(context.Background(), task) if tt.wantErr { require.NotNil(t, err) @@ -502,6 +595,47 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { dbFn: func(args *args) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + }, + inputSubs: []datastore.Subscription{ + { + UID: "123", + FilterConfig: &datastore.FilterConfiguration{ + Filter: datastore.FilterSchema{ + Body: map[string]interface{}{"person.age": 10}, + }, + }, + }, + { + UID: "1234", + FilterConfig: &datastore.FilterConfiguration{ + Filter: datastore.FilterSchema{ + Body: map[string]interface{}{}, + }, + }, + }, + }, + wantSubs: []datastore.Subscription{ + { + UID: "123", + }, + { + UID: "1234", + }, + }, + }, + { + name: "Should skip filter for advanced subscriptions license check failed", + payload: map[string]interface{}{ + "person": map[string]interface{}{ + "age": 10, + }, + }, + dbFn: func(args *args) { + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(false) }, inputSubs: []datastore.Subscription{ { @@ -541,6 +675,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -573,6 +710,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -611,6 +751,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -649,6 +792,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -692,6 +838,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { dbFn: func(args *args) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(4).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -739,6 +888,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(4).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -795,6 +947,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -842,7 +997,7 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { payload, err := json.Marshal(tt.payload) require.NoError(t, err) - subs, err := matchSubscriptionsUsingFilter(context.Background(), &datastore.Event{Data: payload}, args.subRepo, tt.inputSubs, false) + subs, err := matchSubscriptionsUsingFilter(context.Background(), &datastore.Event{Data: payload}, args.subRepo, args.licenser, tt.inputSubs, false) if tt.wantErr { require.NotNil(t, err) return diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index 730925e46e..0e5371ba96 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -9,6 +9,8 @@ import ( "github.com/frain-dev/convoy/pkg/circuit_breaker" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/msgpack" @@ -29,9 +31,9 @@ import ( "github.com/hibiken/asynq" ) -func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, +func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, licenser license.Licenser, projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, - attemptsRepo datastore.DeliveryAttemptsRepository, manager *circuit_breaker.CircuitBreakerManager, + attemptsRepo datastore.DeliveryAttemptsRepository, circuitBreakerManager *circuit_breaker.CircuitBreakerManager, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) (err error) { var data EventDelivery @@ -113,7 +115,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive return &RateLimitError{Err: ErrRateLimit, delay: time.Duration(endpoint.RateLimitDuration) * time.Second} } - err = manager.CanExecute(ctx, endpoint.UID) + err = circuitBreakerManager.CanExecute(ctx, endpoint.UID) if err != nil { return &CircuitBreakerError{Err: err} } @@ -167,7 +169,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive } var httpDuration time.Duration - if endpoint.HttpTimeout == 0 { + if endpoint.HttpTimeout == 0 || !licenser.AdvancedEndpointMgmt() { httpDuration = convoy.HTTP_TIMEOUT_IN_DURATION } else { httpDuration = time.Duration(endpoint.HttpTimeout) * time.Second @@ -200,7 +202,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive eventDelivery.LatencySeconds = time.Since(eventDelivery.GetLatencyStartTime()).Seconds() // register latency - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(licenser) mm.RecordLatency(eventDelivery) } else { @@ -229,10 +231,12 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive log.WithError(err).Error("Failed to reactivate endpoint after successful retry") } - // send endpoint reactivation notification - err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, false, resp.Error, string(resp.Body), resp.StatusCode) - if err != nil { - log.FromContext(ctx).WithError(err).Error("failed to send notification") + if licenser.AdvancedEndpointMgmt() { + // send endpoint reactivation notification + err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, false, resp.Error, string(resp.Body), resp.StatusCode) + if err != nil { + log.FromContext(ctx).WithError(err).Error("failed to send notification") + } } } @@ -268,10 +272,12 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive log.WithError(err).Error("failed to deactivate endpoint after failed retry") } - // send endpoint deactivation notification - err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, true, resp.Error, string(resp.Body), resp.StatusCode) - if err != nil { - log.WithError(err).Error("failed to send notification") + if licenser.AdvancedEndpointMgmt() { + // send endpoint deactivation notification + err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, true, resp.Error, string(resp.Body), resp.StatusCode) + if err != nil { + log.WithError(err).Error("failed to send notification") + } } } } diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 89b4f4c3c4..c9fdb7007d 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -3,11 +3,15 @@ package task import ( "context" "encoding/json" + "testing" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/net" cb "github.com/frain-dev/convoy/pkg/circuit_breaker" "github.com/frain-dev/convoy/pkg/clock" "github.com/stretchr/testify/require" - "testing" + "time" "github.com/frain-dev/convoy" @@ -29,7 +33,7 @@ func TestProcessEventDelivery(t *testing.T) { cfgPath string expectedError error msg *datastore.EventDelivery - dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository) + dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository, license.Licenser) nFn func() func() }{ { @@ -39,7 +43,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { m.EXPECT(). FindEventDeliveryByIDSlim(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.EventDelivery{ @@ -69,7 +73,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ RateLimit: 10, @@ -105,7 +109,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -187,7 +191,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ Secrets: []datastore.Secret{ @@ -253,6 +257,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -272,7 +279,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -338,6 +345,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -357,7 +367,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -423,6 +433,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -442,7 +455,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -510,6 +523,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -529,7 +545,94 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { + a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.Endpoint{ + ProjectID: "123", + Secrets: []datastore.Secret{ + {Value: "secret"}, + }, + RateLimit: 10, + RateLimitDuration: 60, + Status: datastore.ActiveEndpointStatus, + }, nil).Times(1) + + r.EXPECT().AllowWithDuration(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + + m.EXPECT(). + FindEventDeliveryByIDSlim(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.EventDelivery{ + Status: datastore.ScheduledEventStatus, + Metadata: &datastore.Metadata{ + Data: []byte(`{"event": "invoice.completed"}`), + Raw: `{"event": "invoice.completed"}`, + NumTrials: 4, + RetryLimit: 3, + IntervalSeconds: 20, + }, + }, nil).Times(1) + + m.EXPECT(). + UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + o.EXPECT(). + FetchProjectByID(gomock.Any(), gomock.Any()). + Return(&datastore.Project{ + LogoURL: "", + Config: &datastore.ProjectConfig{ + Signature: &datastore.SignatureConfiguration{ + Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Versions: []datastore.SignatureVersion{ + { + UID: "abc", + Hash: "SHA256", + Encoding: datastore.HexEncoding, + }, + }, + }, + SSL: &datastore.DefaultSSLConfig, + Strategy: &datastore.StrategyConfiguration{ + Type: datastore.LinearStrategyProvider, + Duration: 60, + RetryCount: 1, + }, + RateLimit: &datastore.DefaultRateLimitConfig, + DisableEndpoint: true, + }, + }, nil).Times(1) + + a.EXPECT().UpdateEndpointStatus(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + d.EXPECT().CreateDeliveryAttempt(gomock.Any(), gomock.Any()).Times(1) + + m.EXPECT(). + UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) + }, + nFn: func() func() { + httpmock.Activate() + + httpmock.RegisterResponder("POST", "https://google.com", + httpmock.NewStringResponder(200, ``)) + + return func() { + httpmock.DeactivateAndReset() + } + }, + }, + { + name: "Manual retry - disable endpoint - success - advanced endpoint mgmt false", + cfgPath: "./testdata/Config/basic-convoy-disable-endpoint.json", + expectedError: nil, + msg: &datastore.EventDelivery{ + UID: "", + }, + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -594,6 +697,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) }, nFn: func() func() { httpmock.Activate() @@ -613,7 +719,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -683,6 +789,9 @@ func TestProcessEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -702,7 +811,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -773,6 +882,9 @@ func TestProcessEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -802,6 +914,9 @@ func TestProcessEventDelivery(t *testing.T) { q := mocks.NewMockQueuer(ctrl) rateLimiter := mocks.NewMockRateLimiter(ctrl) attemptsRepo := mocks.NewMockDeliveryAttemptsRepository(ctrl) + licenser := mocks.NewMockLicenser(ctrl) + + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) err := config.LoadConfig(tc.cfgPath) if err != nil { @@ -824,10 +939,10 @@ func TestProcessEventDelivery(t *testing.T) { } if tc.dbFn != nil { - tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo) + tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo, licenser) } - dispatcher, err := net.NewDispatcher("", false) + dispatcher, err := net.NewDispatcher("", licenser, false) require.NoError(t, err) mockStore := cb.NewTestStore() @@ -849,7 +964,7 @@ func TestProcessEventDelivery(t *testing.T) { cb.ConfigOption(breakerConfig), ) - processFn := ProcessEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) + processFn := ProcessEventDelivery(endpointRepo, msgRepo, licenser, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) payload := EventDelivery{ EventDeliveryID: tc.msg.UID, diff --git a/worker/task/process_meta_event.go b/worker/task/process_meta_event.go index 788f537f12..57263bab71 100644 --- a/worker/task/process_meta_event.go +++ b/worker/task/process_meta_event.go @@ -28,7 +28,7 @@ type MetaEvent struct { ProjectID string } -func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo datastore.MetaEventRepository) func(context.Context, *asynq.Task) error { +func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo datastore.MetaEventRepository, dispatch *net.Dispatcher) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var data MetaEvent @@ -73,7 +73,7 @@ func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo dat delayDuration := retrystrategies.NewRetryStrategyFromMetadata(*metaEvent.Metadata).NextDuration(metaEvent.Metadata.NumTrials) - resp, err := sendUrlRequest(ctx, project, metaEvent) + resp, err := sendUrlRequest(ctx, project, metaEvent, dispatch) metaEvent.Metadata.NumTrials++ if resp != nil { @@ -120,19 +120,12 @@ func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo dat } } -func sendUrlRequest(ctx context.Context, project *datastore.Project, metaEvent *datastore.MetaEvent) (*net.Response, error) { +func sendUrlRequest(ctx context.Context, project *datastore.Project, metaEvent *datastore.MetaEvent, dispatch *net.Dispatcher) (*net.Response, error) { cfg, err := config.Get() if err != nil { return nil, err } - httpDuration := convoy.HTTP_TIMEOUT_IN_DURATION - dispatch, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, project.Config.SSL.EnforceSecureEndpoints) - if err != nil { - log.WithError(err).Error("error occurred while creating http client") - return nil, err - } - sig := &signature.Signature{ Payload: json.RawMessage(metaEvent.Metadata.Raw), Schemes: []signature.Scheme{ @@ -152,6 +145,7 @@ func sendUrlRequest(ctx context.Context, project *datastore.Project, metaEvent * url := project.Config.MetaEvent.URL + httpDuration := convoy.HTTP_TIMEOUT_IN_DURATION resp, err := dispatch.SendRequest(ctx, url, string(convoy.HttpPost), sig.Payload, "X-Convoy-Signature", header, int64(cfg.MaxResponseSize), httpheader.HTTPHeader{}, dedup.GenerateChecksum(metaEvent.UID), httpDuration) if err != nil { return nil, err diff --git a/worker/task/process_meta_event_test.go b/worker/task/process_meta_event_test.go index 9dc6391dbb..c35c0111ec 100644 --- a/worker/task/process_meta_event_test.go +++ b/worker/task/process_meta_event_test.go @@ -6,6 +6,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + + "github.com/frain-dev/convoy/net" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" @@ -129,8 +133,13 @@ func TestProcessMetaEvent(t *testing.T) { metaEventRepo := mocks.NewMockMetaEventRepository(ctrl) projectRepo := mocks.NewMockProjectRepository(ctrl) + licenser := mocks.NewMockLicenser(ctrl) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) + + dispatcher, err := net.NewDispatcher("", licenser, false) + require.NoError(t, err) - err := config.LoadConfig(tc.cfgPath) + err = config.LoadConfig(tc.cfgPath) if err != nil { t.Errorf("failed to load config file: %v", err) } @@ -144,7 +153,7 @@ func TestProcessMetaEvent(t *testing.T) { tc.dbFn(metaEventRepo, projectRepo) } - processFn := ProcessMetaEvent(projectRepo, metaEventRepo) + processFn := ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher) payload := MetaEvent{ MetaEventID: tc.msg.MetaEventID, ProjectID: tc.msg.ProjectID, diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index f531fd43a1..24b1ba4ef3 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -4,11 +4,14 @@ import ( "context" "encoding/json" "fmt" + "testing" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/net" cb "github.com/frain-dev/convoy/pkg/circuit_breaker" "github.com/frain-dev/convoy/pkg/clock" "github.com/stretchr/testify/require" - "testing" "time" "github.com/frain-dev/convoy" @@ -30,7 +33,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { cfgPath string expectedError error msg *datastore.EventDelivery - dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository) + dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository, license.Licenser) nFn func() func() }{ { @@ -40,7 +43,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { m.EXPECT(). FindEventDeliveryByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.EventDelivery{ @@ -61,6 +64,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { project := &datastore.Project{UID: "project-id-1"} o.EXPECT().FetchProjectByID(gomock.Any(), "project-id-1").Times(1).Return(project, nil) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, }, { @@ -70,7 +76,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ RateLimit: 10, @@ -97,6 +103,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, }, { @@ -106,7 +115,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -160,6 +169,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { d.EXPECT().CreateDeliveryAttempt(gomock.Any(), gomock.Any()).Times(1) + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) + m.EXPECT(). UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) @@ -186,7 +198,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ Secrets: []datastore.Secret{ @@ -223,7 +235,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -252,6 +264,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -271,7 +286,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -308,7 +323,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -319,7 +334,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { }, SSL: &datastore.DefaultSSLConfig, Strategy: &datastore.StrategyConfiguration{ - Type: datastore.StrategyProvider("default"), + Type: "default", Duration: 60, RetryCount: 1, }, @@ -337,6 +352,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -356,7 +374,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -393,7 +411,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -422,6 +440,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -441,7 +462,97 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { + a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.Endpoint{ + ProjectID: "123", + Url: "https://google.com?source=giphy", + Secrets: []datastore.Secret{ + {Value: "secret"}, + }, + RateLimit: 10, + RateLimitDuration: 60, + Status: datastore.ActiveEndpointStatus, + }, nil) + + r.EXPECT().AllowWithDuration(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + + m.EXPECT(). + FindEventDeliveryByID(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.EventDelivery{ + Status: datastore.ScheduledEventStatus, + URLQueryParams: "name=ref&category=food", + Metadata: &datastore.Metadata{ + Data: []byte(`{"event": "invoice.completed"}`), + Raw: `{"event": "invoice.completed"}`, + NumTrials: 4, + RetryLimit: 3, + IntervalSeconds: 20, + }, + }, nil).Times(1) + + a.EXPECT(). + UpdateEndpointStatus(gomock.Any(), gomock.Any(), gomock.Any(), datastore.InactiveEndpointStatus). + Return(nil).Times(1) + + o.EXPECT(). + FetchProjectByID(gomock.Any(), gomock.Any()). + Return(&datastore.Project{ + LogoURL: "", + Config: &datastore.ProjectConfig{ + Signature: &datastore.SignatureConfiguration{ + Header: "X-Convoy-Signature", + Versions: []datastore.SignatureVersion{ + { + UID: "abc", + Hash: "SHA256", + Encoding: datastore.HexEncoding, + }, + }, + }, + SSL: &datastore.DefaultSSLConfig, + Strategy: &datastore.StrategyConfiguration{ + Type: datastore.LinearStrategyProvider, + Duration: 60, + RetryCount: 1, + }, + RateLimit: &datastore.DefaultRateLimitConfig, + DisableEndpoint: true, + }, + }, nil).Times(1) + + d.EXPECT().CreateDeliveryAttempt(gomock.Any(), gomock.Any()).Times(1) + + m.EXPECT(). + UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + m.EXPECT(). + UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) + }, + nFn: func() func() { + httpmock.Activate() + + httpmock.RegisterResponder("POST", "https://google.com?category=food&name=ref&source=giphy", + httpmock.NewStringResponder(200, ``)) + + return func() { + httpmock.DeactivateAndReset() + } + }, + }, + { + name: "Manual retry - disable endpoint - success - skip proxy", + cfgPath: "./testdata/Config/basic-convoy.json", + expectedError: nil, + msg: &datastore.EventDelivery{ + UID: "", + }, + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -480,7 +591,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -509,6 +620,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(false) }, nFn: func() func() { httpmock.Activate() @@ -528,7 +642,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -565,7 +679,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -593,6 +707,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -612,7 +729,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -650,7 +767,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -682,6 +799,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -701,7 +821,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -772,6 +892,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -801,6 +924,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { q := mocks.NewMockQueuer(ctrl) rateLimiter := mocks.NewMockRateLimiter(ctrl) attemptsRepo := mocks.NewMockDeliveryAttemptsRepository(ctrl) + licenser := mocks.NewMockLicenser(ctrl) err := config.LoadConfig(tc.cfgPath) if err != nil { @@ -823,10 +947,10 @@ func TestProcessRetryEventDelivery(t *testing.T) { } if tc.dbFn != nil { - tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo) + tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo, licenser) } - dispatcher, err := net.NewDispatcher("", false) + dispatcher, err := net.NewDispatcher("", licenser, false) require.NoError(t, err) mockStore := cb.NewTestStore() diff --git a/worker/task/search_tokenizer.go b/worker/task/search_tokenizer.go index 21ace0ba3a..70f060a204 100644 --- a/worker/task/search_tokenizer.go +++ b/worker/task/search_tokenizer.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" + fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/rdb" "github.com/frain-dev/convoy/pkg/log" "github.com/go-redsync/redsync/v4" @@ -81,6 +82,18 @@ func TokenizerHandler(eventRepo datastore.EventRepository, jobRepo datastore.Job } func tokenize(ctx context.Context, eventRepo datastore.EventRepository, jobRepo datastore.JobRepository, projectId string, interval int) error { + cfg, err := config.Get() + if err != nil { + return err + } + fflag, err := fflag2.NewFFlag(&cfg) + if err != nil { + return nil + } + if !fflag.CanAccessFeature(fflag2.FullTextSearch) { + return fflag2.ErrFeatureNotEnabled + } + // check if a job for a given project is currently running jobs, err := jobRepo.FetchRunningJobsByProjectId(ctx, projectId) if err != nil { diff --git a/worker/task/testdata/Config/basic-convoy.json b/worker/task/testdata/Config/basic-convoy.json index ab2f3fbcba..28e491efa4 100644 --- a/worker/task/testdata/Config/basic-convoy.json +++ b/worker/task/testdata/Config/basic-convoy.json @@ -47,12 +47,5 @@ "username": "apikey", "password": "", "from": "support@frain.dev" - }, - "search": { - "type": "typesense", - "typesense": { - "host": "http://localhost:8108", - "api_key": "convoy" - } } } From 842080ceed91cd0c694485ec7747e12d98ad7035 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Thu, 5 Sep 2024 16:43:35 +0200 Subject: [PATCH 19/48] chore: fix lint error --- worker/task/process_event_delivery.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index 9481226663..6af8064e84 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -12,9 +12,6 @@ import ( "github.com/frain-dev/convoy/internal/pkg/license" - - "github.com/frain-dev/convoy/internal/pkg/metrics" - "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/msgpack" From abe8ada2f5d231198a838dbd1e83c3055cbec8be Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Thu, 5 Sep 2024 17:44:11 +0200 Subject: [PATCH 20/48] chore: fix lint errors --- .../circuit_breaker_manager.go | 19 +++++++++++-------- pkg/circuit_breaker/metrics.go | 4 ++-- pkg/circuit_breaker/store_test.go | 6 ++++-- worker/task/process_event_delivery_test.go | 2 +- .../task/process_retry_event_delivery_test.go | 2 +- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 821165d8d5..7b4833fcc0 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -134,7 +134,7 @@ func ConfigOption(config *CircuitBreakerConfig) CircuitBreakerOption { } func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + redisCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() keys := make([]string, len(pollResults)) @@ -144,7 +144,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] pollResults[i].Key = key } - res, err := cb.store.GetMany(ctx, keys...) + res, err := cb.store.GetMany(redisCtx, keys...) if err != nil { return err } @@ -224,21 +224,24 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] } func (cb *CircuitBreakerManager) updateCircuitBreakers(ctx context.Context, breakers map[string]CircuitBreaker) (err error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() return cb.store.SetMany(ctx, breakers, time.Duration(cb.config.ObservabilityWindow)*time.Minute) } func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]CircuitBreaker, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + redisCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - keys, err := cb.store.Keys(ctx, prefix) + keys, err := cb.store.Keys(redisCtx, prefix) if err != nil { return nil, err } - res, err := cb.store.GetMany(ctx, keys...) + redisCtx2, cancel2 := context.WithTimeout(ctx, 5*time.Second) + defer cancel2() + + res, err := cb.store.GetMany(redisCtx2, keys...) if err != nil { return nil, err } @@ -301,7 +304,7 @@ func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) err // getCircuitBreaker is used to get fetch the circuit breaker state, // it fails open if the circuit breaker for that key is not found func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() bKey := fmt.Sprintf("%s%s", prefix, key) @@ -327,7 +330,7 @@ func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key stri // GetCircuitBreaker is used to get fetch the circuit breaker state, // it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() bKey := fmt.Sprintf("%s%s", prefix, key) diff --git a/pkg/circuit_breaker/metrics.go b/pkg/circuit_breaker/metrics.go index df93ceeb9a..2d8270515e 100644 --- a/pkg/circuit_breaker/metrics.go +++ b/pkg/circuit_breaker/metrics.go @@ -23,8 +23,8 @@ var ( ) ) -// Call updateMetrics in the sampleStore method after updating each circuit breaker -func (cb *CircuitBreakerManager) updateMetrics(breaker CircuitBreaker) { +func (cb *CircuitBreakerManager) UpdateMetrics(breaker CircuitBreaker) { + // todo(raymond) call UpdateMetrics in the sampleStore method after updating each circuit breaker circuitBreakerState.WithLabelValues(breaker.Key).Set(float64(breaker.State)) circuitBreakerRequests.WithLabelValues(breaker.Key, "success").Add(float64(breaker.TotalSuccesses)) circuitBreakerRequests.WithLabelValues(breaker.Key, "failure").Add(float64(breaker.TotalFailures)) diff --git a/pkg/circuit_breaker/store_test.go b/pkg/circuit_breaker/store_test.go index 6b599b2bcd..5eb6585f23 100644 --- a/pkg/circuit_breaker/store_test.go +++ b/pkg/circuit_breaker/store_test.go @@ -277,13 +277,15 @@ func TestTestStore_Concurrency(t *testing.T) { // Test concurrent reads and writes go func() { for i := 0; i < 100; i++ { - store.SetOne(ctx, "key", CircuitBreaker{Key: "key", State: StateClosed}, time.Minute) + err := store.SetOne(ctx, "key", CircuitBreaker{Key: "key", State: StateClosed}, time.Minute) + require.NoError(t, err) } }() go func() { for i := 0; i < 100; i++ { - store.GetOne(ctx, "key") + _, err := store.GetOne(ctx, "key") + require.NoError(t, err) } }() diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index cb36d67282..435f83706a 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -14,7 +14,6 @@ import ( "time" - "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth/realm_chain" "github.com/frain-dev/convoy/datastore" @@ -964,6 +963,7 @@ func TestProcessEventDelivery(t *testing.T) { cb.ClockOption(mockClock), cb.ConfigOption(breakerConfig), ) + require.NoError(t, err) processFn := ProcessEventDelivery(endpointRepo, msgRepo, licenser, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 720598c3a0..3566c5c875 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/require" "time" - "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth/realm_chain" "github.com/frain-dev/convoy/datastore" @@ -1062,6 +1061,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { cb.ClockOption(mockClock), cb.ConfigOption(breakerConfig), ) + require.NoError(t, err) processFn := ProcessRetryEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) From 97f427cc8d3c0bfe7797a5216e231c852ec56ac1 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 6 Sep 2024 02:54:10 +0200 Subject: [PATCH 21/48] chore: fix test --- go.mod | 2 +- pkg/circuit_breaker/store_test.go | 11 +++++++++-- worker/task/retention_policies_test.go | 11 +++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56257ffc23..ffc79d0beb 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/keygen-sh/keygen-go/v3 v3.2.0 github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 + github.com/mitchellh/mapstructure v1.5.0 github.com/mixpanel/mixpanel-go v1.2.1 github.com/newrelic/go-agent/v3 v3.20.4 github.com/oklog/ulid/v2 v2.1.0 @@ -172,7 +173,6 @@ require ( github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/buildkit v0.15.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect diff --git a/pkg/circuit_breaker/store_test.go b/pkg/circuit_breaker/store_test.go index 5eb6585f23..4c718029cb 100644 --- a/pkg/circuit_breaker/store_test.go +++ b/pkg/circuit_breaker/store_test.go @@ -2,6 +2,8 @@ package circuit_breaker import ( "context" + "fmt" + "sync" "testing" "time" @@ -273,22 +275,27 @@ func TestTestStore_SetMany(t *testing.T) { func TestTestStore_Concurrency(t *testing.T) { store := NewTestStore() ctx := context.Background() + wg := &sync.WaitGroup{} // Test concurrent reads and writes go func() { for i := 0; i < 100; i++ { - err := store.SetOne(ctx, "key", CircuitBreaker{Key: "key", State: StateClosed}, time.Minute) + key := fmt.Sprintf("key_%d", i) + wg.Add(1) + err := store.SetOne(ctx, key, CircuitBreaker{Key: key, State: StateClosed}, time.Minute) require.NoError(t, err) } }() go func() { for i := 0; i < 100; i++ { - _, err := store.GetOne(ctx, "key") + _, err := store.GetOne(ctx, fmt.Sprintf("key_%d", i)) require.NoError(t, err) + wg.Done() } }() // If there's a race condition, this test might panic or deadlock time.Sleep(100 * time.Millisecond) + wg.Wait() } diff --git a/worker/task/retention_policies_test.go b/worker/task/retention_policies_test.go index 6d80c8ad51..83a37ca313 100644 --- a/worker/task/retention_policies_test.go +++ b/worker/task/retention_policies_test.go @@ -3,6 +3,7 @@ package task import ( "context" "fmt" + "github.com/lib/pq" "os" "testing" "time" @@ -447,6 +448,16 @@ func seedConfiguration(db database.Database) (*datastore.Configuration, error) { Policy: "72h", IsRetentionPolicyEnabled: true, }, + CircuitBreakerConfig: &datastore.CircuitBreakerConfig{ + SampleRate: 2, + ErrorTimeout: 30, + FailureThreshold: 0.1, + FailureCount: 3, + SuccessThreshold: 1, + ObservabilityWindow: 5, + NotificationThresholds: pq.Int64Array{10}, + ConsecutiveFailureThreshold: 10, + }, } // Seed Data From 71bb29716afe4e566ec20982c12e27465269a8d2 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 16 Sep 2024 15:51:27 +0200 Subject: [PATCH 22/48] patch: remove concurrent write path --- pkg/circuit_breaker/store_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/circuit_breaker/store_test.go b/pkg/circuit_breaker/store_test.go index 4c718029cb..c3a606a03f 100644 --- a/pkg/circuit_breaker/store_test.go +++ b/pkg/circuit_breaker/store_test.go @@ -277,15 +277,13 @@ func TestTestStore_Concurrency(t *testing.T) { ctx := context.Background() wg := &sync.WaitGroup{} - // Test concurrent reads and writes - go func() { - for i := 0; i < 100; i++ { - key := fmt.Sprintf("key_%d", i) - wg.Add(1) - err := store.SetOne(ctx, key, CircuitBreaker{Key: key, State: StateClosed}, time.Minute) - require.NoError(t, err) - } - }() + // Test concurrent reads + for i := 0; i < 100; i++ { + wg.Add(1) + key := fmt.Sprintf("key_%d", i) + err := store.SetOne(ctx, key, CircuitBreaker{Key: key, State: StateClosed}, time.Minute) + require.NoError(t, err) + } go func() { for i := 0; i < 100; i++ { From ea55312bc4830791bbecf82e9b1fb5710413a856 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 17 Sep 2024 16:05:02 +0200 Subject: [PATCH 23/48] feat: put the circuit breaker behind the license; --- internal/pkg/license/keygen/feature.go | 1 + internal/pkg/license/keygen/keygen.go | 8 +++++ internal/pkg/license/keygen/keygen_test.go | 5 +++ internal/pkg/license/license.go | 1 + internal/pkg/license/noop/noop.go | 2 ++ mocks/license.go | 14 +++++++++ .../circuit_breaker_manager.go | 4 +++ worker/task/process_event_delivery.go | 31 +++++++++++++++---- 8 files changed, 60 insertions(+), 6 deletions(-) diff --git a/internal/pkg/license/keygen/feature.go b/internal/pkg/license/keygen/feature.go index 678ea5bc1b..8f7a2bdc2c 100644 --- a/internal/pkg/license/keygen/feature.go +++ b/internal/pkg/license/keygen/feature.go @@ -24,6 +24,7 @@ const ( PortalLinks Feature = "PORTAL_LINKS" ConsumerPoolTuning Feature = "CONSUMER_POOL_TUNING" AdvancedWebhookFiltering Feature = "ADVANCED_WEBHOOK_FILTERING" + CircuitBreaking Feature = "CIRCUIT_BREAKING" ) const ( diff --git a/internal/pkg/license/keygen/keygen.go b/internal/pkg/license/keygen/keygen.go index 981583472f..1712fa61f8 100644 --- a/internal/pkg/license/keygen/keygen.go +++ b/internal/pkg/license/keygen/keygen.go @@ -443,6 +443,14 @@ func (k *Licenser) AdvancedWebhookFiltering() bool { return ok } +func (k *Licenser) CircuitBreaking() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[CircuitBreaking] + return ok +} + func (k *Licenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { // only these guys have dynamic limits for now for f := range k.featureList { diff --git a/internal/pkg/license/keygen/keygen_test.go b/internal/pkg/license/keygen/keygen_test.go index a14738f807..492c552ada 100644 --- a/internal/pkg/license/keygen/keygen_test.go +++ b/internal/pkg/license/keygen/keygen_test.go @@ -96,6 +96,11 @@ func TestKeygenLicenserBoolMethods(t *testing.T) { k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookFiltering: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} require.False(t, k.AdvancedWebhookFiltering()) + k = Licenser{featureList: map[Feature]*Properties{CircuitBreaking: {}}, license: &keygen.License{}} + require.True(t, k.CircuitBreaking()) + k = Licenser{featureList: map[Feature]*Properties{CircuitBreaking: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.CircuitBreaking()) + k = Licenser{enabledProjects: map[string]bool{ "12345": true, }} diff --git a/internal/pkg/license/license.go b/internal/pkg/license/license.go index faf6a693a1..495f9f726c 100644 --- a/internal/pkg/license/license.go +++ b/internal/pkg/license/license.go @@ -21,6 +21,7 @@ type Licenser interface { PortalLinks() bool ConsumerPoolTuning() bool AdvancedWebhookFiltering() bool + CircuitBreaking() bool // need more fleshing out AdvancedRetentionPolicy() bool diff --git a/internal/pkg/license/noop/noop.go b/internal/pkg/license/noop/noop.go index 6cc613e271..6bc94ac579 100644 --- a/internal/pkg/license/noop/noop.go +++ b/internal/pkg/license/noop/noop.go @@ -98,3 +98,5 @@ func (Licenser) AdvancedWebhookFiltering() bool { func (Licenser) PortalLinks() bool { return true } + +func (Licenser) CircuitBreaking() bool { return true } diff --git a/mocks/license.go b/mocks/license.go index 29d8166aaa..7738b2541c 100644 --- a/mocks/license.go +++ b/mocks/license.go @@ -150,6 +150,20 @@ func (mr *MockLicenserMockRecorder) CanExportPrometheusMetrics() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanExportPrometheusMetrics", reflect.TypeOf((*MockLicenser)(nil).CanExportPrometheusMetrics)) } +// CircuitBreaking mocks base method. +func (m *MockLicenser) CircuitBreaking() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CircuitBreaking") + ret0, _ := ret[0].(bool) + return ret0 +} + +// CircuitBreaking indicates an expected call of CircuitBreaking. +func (mr *MockLicenserMockRecorder) CircuitBreaking() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CircuitBreaking", reflect.TypeOf((*MockLicenser)(nil).CircuitBreaking)) +} + // ConsumerPoolTuning mocks base method. func (m *MockLicenser) ConsumerPoolTuning() bool { m.ctrl.T.Helper() diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 7b4833fcc0..767e97923a 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -378,6 +378,10 @@ func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc P return nil } +func (cb *CircuitBreakerManager) GetConfig() CircuitBreakerConfig { + return *cb.config +} + func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { ticker := time.NewTicker(time.Duration(cb.config.SampleRate) * time.Second) defer ticker.Stop() diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index 6af8064e84..a9709f7c0e 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -116,9 +116,28 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive return &RateLimitError{Err: ErrRateLimit, delay: time.Duration(endpoint.RateLimitDuration) * time.Second} } - err = circuitBreakerManager.CanExecute(ctx, endpoint.UID) - if err != nil { - return &CircuitBreakerError{Err: err} + if licenser.CircuitBreaking() { + breakerErr := circuitBreakerManager.CanExecute(ctx, endpoint.UID) + if breakerErr != nil { + return &CircuitBreakerError{Err: breakerErr} + } + + // check the circuit breaker state so we can disable the endpoint + cb, breakerErr := circuitBreakerManager.GetCircuitBreaker(ctx, endpoint.UID) + if breakerErr != nil { + return &CircuitBreakerError{Err: breakerErr} + } + + if cb != nil { + if cb.ConsecutiveFailures > circuitBreakerManager.GetConfig().ConsecutiveFailureThreshold { + endpointStatus := datastore.InactiveEndpointStatus + + breakerErr = endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) + if breakerErr != nil { + log.WithError(breakerErr).Error("failed to deactivate endpoint after failed retry") + } + } + } } err = eventDeliveryRepo.UpdateStatusOfEventDelivery(ctx, project.UID, *eventDelivery, datastore.ProcessingEventStatus) @@ -225,7 +244,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive log.Errorf("%s failed. Reason: %s", eventDelivery.UID, err) } - if done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint { + if done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint && !licenser.CircuitBreaking() { endpointStatus := datastore.ActiveEndpointStatus err := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) if err != nil { @@ -241,7 +260,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive } } - if !done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint { + if !done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint && !licenser.CircuitBreaking() { endpointStatus := datastore.InactiveEndpointStatus err := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) if err != nil { @@ -265,7 +284,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive eventDelivery.Status = datastore.FailureEventStatus } - if endpoint.Status != datastore.PendingEndpointStatus && project.Config.DisableEndpoint { + if endpoint.Status != datastore.PendingEndpointStatus && project.Config.DisableEndpoint && !licenser.CircuitBreaking() { endpointStatus := datastore.InactiveEndpointStatus err := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) From 85f47028df7c79e3aa7f683b9d3a446ff3c27cb5 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 17 Sep 2024 16:06:38 +0200 Subject: [PATCH 24/48] feat: Change FailureThreshold to an int; change ErrorTimeout to BreakerTimeout; added config validation tests; --- config/config.go | 20 +-- config/config_test.go | 12 +- datastore/models.go | 10 +- internal/notifications/notifications.go | 3 +- pkg/circuit_breaker/circuit_breaker.go | 24 ++-- .../circuit_breaker_manager.go | 24 ++-- .../circuit_breaker_manager_test.go | 120 +++++++++--------- pkg/circuit_breaker/config.go | 44 +++---- pkg/circuit_breaker/config_test.go | 82 ++++++++---- sql/1724236900.sql | 6 +- worker/task/process_event_delivery_test.go | 6 +- .../task/process_retry_event_delivery_test.go | 6 +- worker/task/retention_policies_test.go | 2 +- 13 files changed, 206 insertions(+), 153 deletions(-) diff --git a/config/config.go b/config/config.go index d61082e71c..cd1a25c259 100644 --- a/config/config.go +++ b/config/config.go @@ -77,11 +77,11 @@ var DefaultConfiguration = Configuration{ CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 70, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: []uint64{5, 10}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Auth: AuthConfiguration{ @@ -272,14 +272,14 @@ type RetentionPolicyConfiguration struct { } type CircuitBreakerConfiguration struct { - SampleRate uint64 `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` - FailureCount uint64 `json:"failure_count" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_COUNT"` - ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` - FailureThreshold float64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` - SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` - ObservabilityWindow uint64 `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` - NotificationThresholds []uint64 `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` - ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` + SampleRate uint64 `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` + FailureCount uint64 `json:"failure_count" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_COUNT"` + ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` + FailureThreshold uint64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` + SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` + ObservabilityWindow uint64 `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` + NotificationThresholds [3]uint64 `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` } type AnalyticsConfiguration struct { diff --git a/config/config_test.go b/config/config_test.go index f8c834b1fa..145f57cc39 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -128,11 +128,11 @@ func TestLoadConfig(t *testing.T) { CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 10, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: []uint64{5, 10}, + NotificationThresholds: [3]uint64{5, 10, 15}, ConsecutiveFailureThreshold: 10, }, Server: ServerConfiguration{ @@ -209,11 +209,11 @@ func TestLoadConfig(t *testing.T) { CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 10, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: []uint64{5, 10}, + NotificationThresholds: [3]uint64{5, 10, 15}, ConsecutiveFailureThreshold: 10, }, Redis: RedisConfiguration{ @@ -285,11 +285,11 @@ func TestLoadConfig(t *testing.T) { CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 10, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: []uint64{5, 10}, + NotificationThresholds: [3]uint64{5, 10, 15}, ConsecutiveFailureThreshold: 10, }, Database: DatabaseConfiguration{ diff --git a/datastore/models.go b/datastore/models.go index a4e8776b43..d537482ee2 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -322,11 +322,11 @@ var ( DefaultCircuitBreakerConfiguration = CircuitBreakerConfig{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 70, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: pq.Int64Array{5, 10}, + NotificationThresholds: pq.Int64Array{10, 30, 50}, ConsecutiveFailureThreshold: 10, } ) @@ -1345,14 +1345,14 @@ func (c *Configuration) GetCircuitBreakerConfig() CircuitBreakerConfig { } func (c *Configuration) ToCircuitBreakerConfig() *circuit_breaker.CircuitBreakerConfig { - notificationThresholds := make([]uint64, len(c.CircuitBreakerConfig.NotificationThresholds)) + notificationThresholds := [3]uint64{} for i := range c.CircuitBreakerConfig.NotificationThresholds { notificationThresholds[i] = uint64(c.CircuitBreakerConfig.NotificationThresholds[i]) } return &circuit_breaker.CircuitBreakerConfig{ SampleRate: c.CircuitBreakerConfig.SampleRate, - ErrorTimeout: c.CircuitBreakerConfig.ErrorTimeout, + BreakerTimeout: c.CircuitBreakerConfig.ErrorTimeout, FailureThreshold: c.CircuitBreakerConfig.FailureThreshold, FailureCount: c.CircuitBreakerConfig.FailureCount, SuccessThreshold: c.CircuitBreakerConfig.SuccessThreshold, @@ -1392,7 +1392,7 @@ type OnPremStorage struct { type CircuitBreakerConfig struct { SampleRate uint64 `json:"sample_rate" db:"sample_rate"` ErrorTimeout uint64 `json:"error_timeout" db:"error_timeout"` - FailureThreshold float64 `json:"failure_threshold" db:"failure_threshold"` + FailureThreshold uint64 `json:"failure_threshold" db:"failure_threshold"` FailureCount uint64 `json:"failure_count" db:"failure_count"` SuccessThreshold uint64 `json:"success_threshold" db:"success_threshold"` ObservabilityWindow uint64 `json:"observability_window" db:"observability_window"` diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go index a59558f7bd..9e120faf61 100644 --- a/internal/notifications/notifications.go +++ b/internal/notifications/notifications.go @@ -37,7 +37,8 @@ type SlackNotification struct { // NOTIFICATIONS -func SendEndpointNotification(ctx context.Context, +func SendEndpointNotification( + _ context.Context, endpoint *datastore.Endpoint, project *datastore.Project, status datastore.EndpointStatus, diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index ef88664555..34059e5a4d 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -7,14 +7,22 @@ import ( // CircuitBreaker represents a circuit breaker type CircuitBreaker struct { - Key string `json:"key"` - State State `json:"state"` - Requests uint64 `json:"requests"` - FailureRate float64 `json:"failure_rate"` - WillResetAt time.Time `json:"will_reset_at"` - TotalFailures uint64 `json:"total_failures"` - TotalSuccesses uint64 `json:"total_successes"` - ConsecutiveFailures uint64 `json:"consecutive_failures"` + // Circuit breaker key + Key string `json:"key"` + // Circuit breaker state + State State `json:"state"` + // Number of requests in the observability window + Requests uint64 `json:"requests"` + // Percentage of failures in the observability window + FailureRate float64 `json:"failure_rate"` + // Time after which the circuit breaker will reset + WillResetAt time.Time `json:"will_reset_at"` + // Number of failed requests in the observability window + TotalFailures uint64 `json:"total_failures"` + // Number of successful requests in the observability window + TotalSuccesses uint64 `json:"total_successes"` + // Number of consecutive circuit breaker trips + ConsecutiveFailures uint64 `json:"consecutive_failures"` } func (b *CircuitBreaker) String() (s string, err error) { diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 767e97923a..b86c4ca54e 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -10,6 +10,11 @@ import ( "time" ) +// todo(raymond): add to feature flags +// todo(raymond): notification thresholds are percentages +// todo(raymond): metrics should contain error rate +// todo(raymond): use a guage for failure rate metrics + const prefix = "breaker:" const mutexKey = "convoy:circuit_breaker:mutex" @@ -199,13 +204,14 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] if breaker.Requests == 0 { breaker.FailureRate = 0 } else { - breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.Requests) + breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.Requests) * 100 } if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { breaker.resetCircuitBreaker() - } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && (breaker.FailureRate >= cb.config.FailureThreshold || breaker.TotalFailures >= cb.config.FailureCount) { - breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.ErrorTimeout) * time.Second)) + } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && + (breaker.FailureRate >= float64(cb.config.FailureThreshold) || breaker.TotalFailures >= cb.config.FailureCount) { + breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) } if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { @@ -282,9 +288,9 @@ func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error // CanExecute checks if the circuit breaker for a key will return an error for the current state. // It will not return an error if it is in the closed state or half-open state when the failure -// threshold has not been reached, it will fail-open if the circuit breaker is not found +// threshold has not been reached, it will also fail-open if the circuit breaker is not found. func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) error { - b, err := cb.getCircuitBreaker(ctx, key) + b, err := cb.GetCircuitBreaker(ctx, key) if err != nil { return err } @@ -301,9 +307,9 @@ func (cb *CircuitBreakerManager) CanExecute(ctx context.Context, key string) err return nil } -// getCircuitBreaker is used to get fetch the circuit breaker state, +// GetCircuitBreaker is used to get fetch the circuit breaker state, // it fails open if the circuit breaker for that key is not found -func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { +func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() @@ -327,9 +333,9 @@ func (cb *CircuitBreakerManager) getCircuitBreaker(ctx context.Context, key stri return c, nil } -// GetCircuitBreaker is used to get fetch the circuit breaker state, +// GetCircuitBreakerWithError is used to get fetch the circuit breaker state, // it returns ErrCircuitBreakerNotFound when a circuit breaker for the key is not found -func (cb *CircuitBreakerManager) GetCircuitBreaker(ctx context.Context, key string) (c *CircuitBreaker, err error) { +func (cb *CircuitBreakerManager) GetCircuitBreakerWithError(ctx context.Context, key string) (c *CircuitBreaker, err error) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index f59daf91ed..a65fababa0 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -52,12 +52,12 @@ func TestCircuitBreakerManager(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.1, + BreakerTimeout: 30, + FailureThreshold: 10, FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, + NotificationThresholds: [3]uint64{10}, ConsecutiveFailureThreshold: 10, } @@ -81,7 +81,7 @@ func TestCircuitBreakerManager(t *testing.T) { testClock.AdvanceTime(time.Minute) } - breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) + breaker, innerErr := b.GetCircuitBreakerWithError(ctx, endpointId) require.NoError(t, innerErr) require.Equal(t, breaker.State, StateClosed) @@ -105,12 +105,12 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.1, + BreakerTimeout: 30, + FailureThreshold: 10, FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, + NotificationThresholds: [3]uint64{10}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -160,12 +160,12 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 3, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, + NotificationThresholds: [3]uint64{10}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -196,14 +196,14 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { err = b.sampleStore(ctx, result) require.NoError(t, err) - breaker, innerErr := b.GetCircuitBreaker(ctx, endpointId) + breaker, innerErr := b.GetCircuitBreakerWithError(ctx, endpointId) require.NoError(t, innerErr) require.Equal(t, expectedStates[i], breaker.State, "Iteration %d: expected state %v, got %v", i, expectedStates[i], breaker.State) if i == 2 { // Advance time to trigger the transition to half-open - testClock.AdvanceTime(time.Duration(c.ErrorTimeout+1) * time.Second) + testClock.AdvanceTime(time.Duration(c.BreakerTimeout+1) * time.Second) } else { testClock.AdvanceTime(time.Second * 5) // Advance time by 5 seconds for other iterations } @@ -230,12 +230,12 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 3, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, + NotificationThresholds: [3]uint64{10}, ConsecutiveFailureThreshold: 3, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -254,10 +254,10 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { err = b.sampleStore(ctx, result) require.NoError(t, err) - testClock.AdvanceTime(time.Duration(c.ErrorTimeout+1) * time.Second) + testClock.AdvanceTime(time.Duration(c.BreakerTimeout+1) * time.Second) } - breaker, err := b.GetCircuitBreaker(ctx, endpointId) + breaker, err := b.GetCircuitBreakerWithError(ctx, endpointId) require.NoError(t, err) require.Equal(t, StateOpen, breaker.State) require.Equal(t, uint64(3), breaker.ConsecutiveFailures) @@ -283,12 +283,12 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 3, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10}, + NotificationThresholds: [3]uint64{10}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -308,18 +308,18 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { err = b.sampleStore(ctx, results) require.NoError(t, err) - testClock.AdvanceTime(time.Duration(c.ErrorTimeout+1) * time.Second) + testClock.AdvanceTime(time.Duration(c.BreakerTimeout+1) * time.Second) } - breaker1, err := b.GetCircuitBreaker(ctx, endpoint1) + breaker1, err := b.GetCircuitBreakerWithError(ctx, endpoint1) require.NoError(t, err) require.Equal(t, StateOpen, breaker1.State) - breaker2, err := b.GetCircuitBreaker(ctx, endpoint2) + breaker2, err := b.GetCircuitBreakerWithError(ctx, endpoint2) require.NoError(t, err) require.Equal(t, StateClosed, breaker2.State) - breaker3, err := b.GetCircuitBreaker(ctx, endpoint3) + breaker3, err := b.GetCircuitBreakerWithError(ctx, endpoint3) require.NoError(t, err) require.Equal(t, StateClosed, breaker3.State) } @@ -329,12 +329,12 @@ func TestCircuitBreakerManager_Config(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -386,12 +386,12 @@ func TestCircuitBreakerManager_Config(t *testing.T) { func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -427,12 +427,12 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -453,14 +453,14 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { require.NoError(t, err) // Check if circuit breakers were created and updated correctly - cb1, err := manager.GetCircuitBreaker(ctx, "test1") + cb1, err := manager.GetCircuitBreakerWithError(ctx, "test1") require.NoError(t, err) require.Equal(t, StateClosed, cb1.State) require.Equal(t, uint64(10), cb1.Requests) require.Equal(t, uint64(3), cb1.TotalFailures) require.Equal(t, uint64(7), cb1.TotalSuccesses) - cb2, err := manager.GetCircuitBreaker(ctx, "test2") + cb2, err := manager.GetCircuitBreakerWithError(ctx, "test2") require.NoError(t, err) require.Equal(t, StateOpen, cb2.State) require.Equal(t, uint64(10), cb2.Requests) @@ -473,12 +473,12 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -511,14 +511,14 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { require.NoError(t, err) // Check if circuit breakers were updated in the store - cb1, err := manager.GetCircuitBreaker(ctx, "test1") + cb1, err := manager.GetCircuitBreakerWithError(ctx, "test1") require.NoError(t, err) require.Equal(t, StateClosed, cb1.State) require.Equal(t, uint64(10), cb1.Requests) require.Equal(t, uint64(3), cb1.TotalFailures) require.Equal(t, uint64(7), cb1.TotalSuccesses) - cb2, err := manager.GetCircuitBreaker(ctx, "test2") + cb2, err := manager.GetCircuitBreakerWithError(ctx, "test2") require.NoError(t, err) require.Equal(t, StateOpen, cb2.State) require.Equal(t, uint64(10), cb2.Requests) @@ -531,12 +531,12 @@ func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -588,12 +588,12 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -667,12 +667,12 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -686,7 +686,7 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { ctx := context.Background() t.Run("Circuit Breaker Not Found", func(t *testing.T) { - _, err := manager.GetCircuitBreaker(ctx, "non_existent") + _, err := manager.GetCircuitBreakerWithError(ctx, "non_existent") require.Equal(t, ErrCircuitBreakerNotFound, err) }) @@ -701,7 +701,7 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { err := manager.store.SetOne(ctx, "breaker:test_cb", cb, time.Minute) require.NoError(t, err) - retrievedCB, err := manager.GetCircuitBreaker(ctx, "test_cb") + retrievedCB, err := manager.GetCircuitBreakerWithError(ctx, "test_cb") require.NoError(t, err) require.Equal(t, cb.Key, retrievedCB.Key) require.Equal(t, cb.State, retrievedCB.State) @@ -716,12 +716,12 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -746,14 +746,14 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { require.NoError(t, err) // Check if circuit breakers were created and updated correctly - cb1, err := manager.GetCircuitBreaker(ctx, "test1") + cb1, err := manager.GetCircuitBreakerWithError(ctx, "test1") require.NoError(t, err) require.Equal(t, StateClosed, cb1.State) require.Equal(t, uint64(10), cb1.Requests) require.Equal(t, uint64(3), cb1.TotalFailures) require.Equal(t, uint64(7), cb1.TotalSuccesses) - cb2, err := manager.GetCircuitBreaker(ctx, "test2") + cb2, err := manager.GetCircuitBreakerWithError(ctx, "test2") require.NoError(t, err) require.Equal(t, StateOpen, cb2.State) require.Equal(t, uint64(10), cb2.Requests) @@ -787,12 +787,12 @@ func TestCircuitBreakerManager_Start(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -820,7 +820,7 @@ func TestCircuitBreakerManager_Start(t *testing.T) { time.Sleep(2500 * time.Millisecond) // Check if the circuit breaker was updated - cb, err := manager.GetCircuitBreaker(ctx, "test") + cb, err := manager.GetCircuitBreakerWithError(ctx, "test") require.NoError(t, err) require.NotNil(t, cb) require.Equal(t, uint64(10), cb.Requests) diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go index ff37b24874..f7f58a428f 100644 --- a/pkg/circuit_breaker/config.go +++ b/pkg/circuit_breaker/config.go @@ -11,13 +11,13 @@ type CircuitBreakerConfig struct { // is polled to determine the number successful and failed requests SampleRate uint64 `json:"sample_rate"` - // ErrorTimeout is the time (in seconds) after which a circuit breaker goes + // BreakerTimeout is the time (in seconds) after which a circuit breaker goes // into the half-open state from the open state - ErrorTimeout uint64 `json:"error_timeout"` + BreakerTimeout uint64 `json:"breaker_timeout"` // FailureThreshold is the % of failed requests in the observability window // after which the breaker will go into the open state - FailureThreshold float64 `json:"failure_threshold"` + FailureThreshold uint64 `json:"failure_threshold"` // FailureCount total number of failed requests in the observability window FailureCount uint64 `json:"failure_count"` @@ -31,7 +31,7 @@ type CircuitBreakerConfig struct { ObservabilityWindow uint64 `json:"observability_window"` // NotificationThresholds These are the error counts after which we will send out notifications. - NotificationThresholds []uint64 `json:"notification_thresholds"` + NotificationThresholds [3]uint64 `json:"notification_thresholds"` // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. // E.g., after 10 consecutive transitions from half-open → open we should disable it. @@ -46,13 +46,13 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - if c.ErrorTimeout == 0 { - errs.WriteString("ErrorTimeout must be greater than 0") + if c.BreakerTimeout == 0 { + errs.WriteString("BreakerTimeout must be greater than 0") errs.WriteString("; ") } - if c.FailureThreshold < 0 || c.FailureThreshold > 1 { - errs.WriteString("FailureThreshold must be between 0 and 1") + if c.FailureThreshold < 0 || c.FailureThreshold > 100 { + errs.WriteString("FailureThreshold must be between 0 and 100") errs.WriteString("; ") } @@ -77,22 +77,22 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - if len(c.NotificationThresholds) == 0 { - errs.WriteString("NotificationThresholds must contain at least one threshold") - errs.WriteString("; ") - } else { - for i := 0; i < len(c.NotificationThresholds); i++ { - if c.NotificationThresholds[i] == 0 { - errs.WriteString(fmt.Sprintf("Notification thresholds at index [%d] = %d must be greater than 0", i, c.NotificationThresholds[i])) - errs.WriteString("; ") - } + for i := 0; i < len(c.NotificationThresholds); i++ { + if c.NotificationThresholds[i] == 0 { + errs.WriteString(fmt.Sprintf("Notification threshold at index [%d] = %d must be greater than 0", i, c.NotificationThresholds[i])) + errs.WriteString("; ") + } + + if c.NotificationThresholds[i] > c.FailureThreshold { + errs.WriteString(fmt.Sprintf("Notification threshold at index [%d] = %d must be less than the failure threshold: %d", i, c.NotificationThresholds[i], c.FailureThreshold)) + errs.WriteString("; ") } + } - for i := 0; i < len(c.NotificationThresholds)-1; i++ { - if c.NotificationThresholds[i] >= c.NotificationThresholds[i+1] { - errs.WriteString("NotificationThresholds should be in ascending order") - errs.WriteString("; ") - } + for i := 0; i < len(c.NotificationThresholds)-1; i++ { + if c.NotificationThresholds[i] >= c.NotificationThresholds[i+1] { + errs.WriteString("NotificationThresholds should be in ascending order") + errs.WriteString("; ") } } diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index 6673c5b673..caae00ef78 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -10,17 +10,18 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { name string config CircuitBreakerConfig wantErr bool + err string }{ { name: "Valid Config", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 5, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, }, wantErr: false, @@ -31,84 +32,121 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 0, }, wantErr: true, + err: "SampleRate must be greater than 0", }, { name: "Invalid ErrorTimeout", config: CircuitBreakerConfig{ - SampleRate: 1, - ErrorTimeout: 0, + SampleRate: 1, + BreakerTimeout: 0, }, wantErr: true, + err: "BreakerTimeout must be greater than 0", }, { name: "Invalid FailureThreshold", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 1.5, + BreakerTimeout: 30, + FailureThreshold: 150, }, wantErr: true, + err: "FailureThreshold must be between 0 and 100", }, { name: "Invalid FailureCount", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 0, }, wantErr: true, + err: "FailureCount must be greater than 0", }, { name: "Invalid SuccessThreshold", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 5, FailureCount: 5, SuccessThreshold: 0, }, wantErr: true, + err: "SuccessThreshold must be greater than 0", }, { name: "Invalid ObservabilityWindow", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 0, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, }, wantErr: true, + err: "ObservabilityWindow must be greater than 0", + }, + { + name: "ObservabilityWindow should be greater than sample rate", + config: CircuitBreakerConfig{ + SampleRate: 200, + BreakerTimeout: 30, + FailureThreshold: 50, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 1, + NotificationThresholds: [3]uint64{10, 20, 30}, + }, + wantErr: true, + err: "ObservabilityWindow must be greater than the SampleRate", }, { name: "Invalid NotificationThresholds", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 5, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{}, + NotificationThresholds: [3]uint64{}, }, wantErr: true, + err: "Notification threshold at index [0] = 0 must be greater than 0", }, { name: "Invalid ConsecutiveFailureThreshold", config: CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 0, }, wantErr: true, + err: "ConsecutiveFailureThreshold must be greater than 0", + }, + { + name: "NotificationThreshold larger than FailureThreshold", + config: CircuitBreakerConfig{ + SampleRate: 1, + BreakerTimeout: 30, + FailureThreshold: 30, + FailureCount: 5, + SuccessThreshold: 2, + ObservabilityWindow: 5, + NotificationThresholds: [3]uint64{30, 50, 60}, + ConsecutiveFailureThreshold: 1, + }, + wantErr: true, + err: "Notification threshold at index [2] = 60 must be less than the failure threshold", }, } @@ -117,8 +155,8 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { err := tt.config.Validate() if tt.wantErr { require.Error(t, err) - } else { - require.NoError(t, err) + require.NotEmpty(t, tt.err) + require.Contains(t, err.Error(), tt.err) } }) } diff --git a/sql/1724236900.sql b/sql/1724236900.sql index 79879db4d4..a52669bee8 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -1,12 +1,12 @@ -- +migrate Up alter table convoy.configurations add column if not exists cb_sample_rate int not null default 30; alter table convoy.configurations add column if not exists cb_error_timeout int not null default 30; -alter table convoy.configurations add column if not exists cb_failure_threshold float not null default 0.1; +alter table convoy.configurations add column if not exists cb_failure_threshold int not null default 70; alter table convoy.configurations add column if not exists cb_failure_count int not null default 1; alter table convoy.configurations add column if not exists cb_success_threshold int not null default 5; alter table convoy.configurations add column if not exists cb_observability_window int not null default 5; -alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[5, 10]; -alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 5; +alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[10, 30, 50]; +alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 10; create index if not exists idx_delivery_attempts_created_at on convoy.delivery_attempts (created_at); create index if not exists idx_delivery_attempts_event_delivery_id_created_at on convoy.delivery_attempts (event_delivery_id, created_at); create index if not exists idx_delivery_attempts_event_delivery_id on convoy.delivery_attempts (event_delivery_id); diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 435f83706a..59c88533ca 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -949,12 +949,12 @@ func TestProcessEventDelivery(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Now()) breakerConfig := &cb.CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 3566c5c875..8003e95565 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -1047,12 +1047,12 @@ func TestProcessRetryEventDelivery(t *testing.T) { mockClock := clock.NewSimulatedClock(time.Now()) breakerConfig := &cb.CircuitBreakerConfig{ SampleRate: 1, - ErrorTimeout: 30, - FailureThreshold: 0.5, + BreakerTimeout: 30, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: []uint64{10, 20, 30}, + NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } diff --git a/worker/task/retention_policies_test.go b/worker/task/retention_policies_test.go index 83a37ca313..79558e3cf3 100644 --- a/worker/task/retention_policies_test.go +++ b/worker/task/retention_policies_test.go @@ -451,7 +451,7 @@ func seedConfiguration(db database.Database) (*datastore.Configuration, error) { CircuitBreakerConfig: &datastore.CircuitBreakerConfig{ SampleRate: 2, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 10, FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, From 9812d531f24cf7e99207f838c38b030f7e922385 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 17 Sep 2024 18:58:29 +0200 Subject: [PATCH 25/48] feat: put circuit breaker behind a feature flag --- cmd/agent/agent.go | 14 +++----- cmd/ff/feature_flags.go | 6 ++-- cmd/hooks/hooks.go | 15 ++++++--- cmd/ingest/ingest.go | 4 +-- cmd/server/server.go | 5 +-- cmd/stream/stream.go | 4 +-- cmd/worker/worker.go | 36 ++++++++++++--------- ee/cmd/domain/domain.go | 2 +- ee/cmd/server/server.go | 7 ++-- internal/pkg/fflag/fflag.go | 14 ++++++-- internal/pkg/fflag/fflag_test.go | 7 ++-- pkg/log/log.go | 4 +-- worker/task/process_event_delivery.go | 5 +-- worker/task/process_retry_event_delivery.go | 31 +++++++++++++++--- worker/task/search_tokenizer.go | 9 +++--- 15 files changed, 94 insertions(+), 69 deletions(-) diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index 0628cb7f78..513b2675ca 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -2,7 +2,6 @@ package agent import ( "context" - "fmt" "os" "os/signal" "time" @@ -114,7 +113,7 @@ func AddAgentCommand(a *cli.App) *cobra.Command { cmd.Flags().Uint32Var(&workerPort, "worker-port", 0, "Worker port") cmd.Flags().Uint32Var(&ingestPort, "ingest-port", 0, "Ingest port") - cmd.Flags().StringVar(&logLevel, "log-level", "", "scheduler log level") + cmd.Flags().StringVar(&logLevel, "log-level", "", "Log level") cmd.Flags().IntVar(&consumerPoolSize, "consumers", -1, "Size of the consumers pool.") cmd.Flags().IntVar(&interval, "interval", 10, "the time interval, measured in seconds to update the in-memory store from the database") cmd.Flags().StringVar(&executionMode, "mode", "", "Execution Mode (one of events, retry and default)") @@ -122,7 +121,7 @@ func AddAgentCommand(a *cli.App) *cobra.Command { return cmd } -func startServerComponent(ctx context.Context, a *cli.App) error { +func startServerComponent(_ context.Context, a *cli.App) error { cfg, err := config.Get() if err != nil { a.Logger.WithError(err).Fatal("Failed to load configuration") @@ -139,10 +138,7 @@ func startServerComponent(ctx context.Context, a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag(&cfg) - if err != nil { - a.Logger.WithError(err).Fatal("failed to create fflag controller") - } + flag := fflag.NewFFlag(&cfg) lo := a.Logger.(*log.Logger) lo.SetPrefix("api server") @@ -172,7 +168,7 @@ func startServerComponent(ctx context.Context, a *cli.App) error { srv.SetHandler(evHandler.BuildDataPlaneRoutes()) - fmt.Printf("Started convoy server in %s\n", time.Since(start)) + log.Info("Started convoy server in %s\n", time.Since(start)) httpConfig := cfg.Server.HTTP if httpConfig.SSL { @@ -181,7 +177,7 @@ func startServerComponent(ctx context.Context, a *cli.App) error { return nil } - fmt.Printf("Starting Convoy Agent on port %v\n", cfg.Server.HTTP.AgentPort) + log.Println("Starting Convoy Agent on port %v\n", cfg.Server.HTTP.AgentPort) go func() { srv.Listen() diff --git a/cmd/ff/feature_flags.go b/cmd/ff/feature_flags.go index af9dc6b5f2..71cf043fc2 100644 --- a/cmd/ff/feature_flags.go +++ b/cmd/ff/feature_flags.go @@ -20,10 +20,8 @@ func AddFeatureFlagsCommand() *cobra.Command { if err != nil { log.WithError(err).Fatalf("Error fetching the config.") } - f, err := fflag2.NewFFlag(&cfg) - if err != nil { - return err - } + + f := fflag2.NewFFlag(&cfg) return f.ListFeatures() }, PersistentPostRun: func(cmd *cobra.Command, args []string) {}, diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index ad46e74162..13a4104f5f 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -560,32 +560,34 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { } - flag, err := fflag2.NewFFlag(c) - if err != nil { - return nil, err - } + flag := fflag2.NewFFlag(c) c.Metrics = config.MetricsConfiguration{ IsEnabled: false, } + if flag.CanAccessFeature(fflag2.Prometheus) { metricsBackend, err := cmd.Flags().GetString("metrics-backend") if err != nil { return nil, err } + if !config.IsStringEmpty(metricsBackend) { c.Metrics = config.MetricsConfiguration{ IsEnabled: false, Backend: config.MetricsBackend(metricsBackend), } + switch c.Metrics.Backend { case config.PrometheusMetricsProvider: sampleTime, err := cmd.Flags().GetUint64("metrics-prometheus-sample-time") if err != nil { return nil, err } + if sampleTime < 1 { return nil, errors.New("metrics-prometheus-sample-time must be non-zero") } + c.Metrics = config.MetricsConfiguration{ IsEnabled: true, Backend: config.MetricsBackend(metricsBackend), @@ -595,14 +597,17 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { } } } else { - log.Warn("No metrics-backend specified") + log.Warn("metrics backend not specified") } + } else { + log.Info(fflag2.ErrPrometheusMetricsNotEnabled) } maxRetrySeconds, err := cmd.Flags().GetUint64("max-retry-seconds") if err != nil { return nil, err } + c.MaxRetrySeconds = maxRetrySeconds return c, nil diff --git a/cmd/ingest/ingest.go b/cmd/ingest/ingest.go index 334f7bdbd1..27b89e6619 100644 --- a/cmd/ingest/ingest.go +++ b/cmd/ingest/ingest.go @@ -65,7 +65,7 @@ func AddIngestCommand(a *cli.App) *cobra.Command { } cmd.Flags().Uint32Var(&ingestPort, "ingest-port", 0, "Ingest port") - cmd.Flags().StringVar(&logLevel, "log-level", "", "ingest log level") + cmd.Flags().StringVar(&logLevel, "log-level", "", "Log level") cmd.Flags().IntVar(&interval, "interval", 10, "the time interval, measured in seconds, at which the database should be polled for new pub sub sources") return cmd @@ -119,7 +119,7 @@ func StartIngest(ctx context.Context, a *cli.App, cfg config.Configuration, inte go ingest.Run() - fmt.Println("Starting Convoy Ingester") + log.Println("Starting Convoy Ingester") return nil } diff --git a/cmd/server/server.go b/cmd/server/server.go index 87d9be3486..972d066db5 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -106,10 +106,7 @@ func startConvoyServer(a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag(&cfg) - if err != nil { - a.Logger.WithError(err).Fatal("failed to create fflag controller") - } + flag := fflag.NewFFlag(&cfg) if cfg.Server.HTTP.Port <= 0 { return errors.New("please provide the HTTP port in the convoy.json file") diff --git a/cmd/stream/stream.go b/cmd/stream/stream.go index 72d37baf48..9a5275aebf 100644 --- a/cmd/stream/stream.go +++ b/cmd/stream/stream.go @@ -136,12 +136,12 @@ func AddStreamCommand(a *cli.App) *cobra.Command { } cmd.Flags().Uint32Var(&socketPort, "socket-port", 5008, "Socket port") - cmd.Flags().StringVar(&logLevel, "log-level", "error", "stream log level") + cmd.Flags().StringVar(&logLevel, "log-level", "", "Log level") return cmd } -func buildCliFlagConfiguration(cmd *cobra.Command) (*config.Configuration, error) { +func buildCliFlagConfiguration(_ *cobra.Command) (*config.Configuration, error) { c := &config.Configuration{} return c, nil diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 3b427ec225..6367963ac6 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -3,13 +3,13 @@ package worker import ( "context" "fmt" + "github.com/frain-dev/convoy/internal/pkg/fflag" "net/http" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/internal/pkg/cli" - fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/loader" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -246,15 +246,22 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - circuitBreakerManager, err := cb.NewCircuitBreakerManager( - cb.ConfigOption(configuration.ToCircuitBreakerConfig()), - cb.StoreOption(cb.NewRedisStore(rd.Client(), clock.NewRealClock())), - cb.ClockOption(clock.NewRealClock())) - if err != nil { - a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") - } + featureFlag := fflag.NewFFlag(&cfg) + var circuitBreakerManager *cb.CircuitBreakerManager - go circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) + if featureFlag.CanAccessFeature(fflag.CircuitBreaker) { + circuitBreakerManager, err = cb.NewCircuitBreakerManager( + cb.ConfigOption(configuration.ToCircuitBreakerConfig()), + cb.StoreOption(cb.NewRedisStore(rd.Client(), clock.NewRealClock())), + cb.ClockOption(clock.NewRealClock())) + if err != nil { + a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") + } + + go circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) + } else { + a.Logger.Warn(fflag.ErrCircuitBreakerNotEnabled) + } consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, @@ -266,6 +273,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte dispatcher, attemptRepo, circuitBreakerManager, + featureFlag, ), newTelemetry) consumer.RegisterHandlers(convoy.CreateEventProcessor, task.ProcessEventCreation( @@ -280,12 +288,14 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.RetryEventProcessor, task.ProcessRetryEventDelivery( endpointRepo, eventDeliveryRepo, + a.Licenser, projectRepo, a.Queue, rateLimiter, dispatcher, attemptRepo, circuitBreakerManager, + featureFlag, ), newTelemetry) consumer.RegisterHandlers(convoy.CreateBroadcastEventProcessor, task.ProcessBroadcastEventCreation( @@ -324,11 +334,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.DailyAnalytics, task.PushDailyTelemetry(lo, a.DB, a.Cache, rd), nil) consumer.RegisterHandlers(convoy.EmailProcessor, task.ProcessEmails(sc), nil) - fflag, err := fflag2.NewFFlag(&cfg) - if err != nil { - return nil - } - if fflag.CanAccessFeature(fflag2.FullTextSearch) && a.Licenser.AdvancedWebhookFiltering() { + if featureFlag.CanAccessFeature(fflag.FullTextSearch) && a.Licenser.AdvancedWebhookFiltering() { consumer.RegisterHandlers(convoy.TokenizeSearch, task.GeneralTokenizerHandler(projectRepo, eventRepo, jobRepo, rd), nil) consumer.RegisterHandlers(convoy.TokenizeSearchForProject, task.TokenizerHandler(eventRepo, jobRepo), nil) } @@ -341,7 +347,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte // start worker consumer.Start() - fmt.Println("Starting Convoy Consumer Pool") + lo.Println("Starting Convoy Consumer Pool") return ctx.Err() } diff --git a/ee/cmd/domain/domain.go b/ee/cmd/domain/domain.go index 3b84df648e..ba77998598 100644 --- a/ee/cmd/domain/domain.go +++ b/ee/cmd/domain/domain.go @@ -121,7 +121,7 @@ func AddDomainCommand(a *cli.App) *cobra.Command { } cmd.Flags().Uint32Var(&domainPort, "domain-port", 5009, "Domain server port") - cmd.Flags().StringVar(&logLevel, "log-level", "error", "Domain server log level") + cmd.Flags().StringVar(&logLevel, "log-level", "", "Domain server log level") return cmd } diff --git a/ee/cmd/server/server.go b/ee/cmd/server/server.go index 3078cc19d5..f42216b53b 100644 --- a/ee/cmd/server/server.go +++ b/ee/cmd/server/server.go @@ -84,7 +84,7 @@ func AddServerCommand(a *cli.App) *cobra.Command { cmd.Flags().StringVar(&apiKeyAuthConfig, "api-auth", "", "API-Key authentication credentials") cmd.Flags().StringVar(&basicAuthConfig, "basic-auth", "", "Basic authentication credentials") - cmd.Flags().StringVar(&logLevel, "log-level", "error", "Log level") + cmd.Flags().StringVar(&logLevel, "log-level", "", "Log level") cmd.Flags().StringVar(&logger, "logger", "info", "Logger") cmd.Flags().StringVar(&proxy, "proxy", "", "HTTP Proxy") cmd.Flags().StringVar(&env, "env", "development", "Convoy environment") @@ -136,10 +136,7 @@ func StartConvoyServer(a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag(&cfg) - if err != nil { - a.Logger.WithError(err).Fatal("failed to create fflag controller") - } + flag := fflag.NewFFlag(&cfg) if cfg.Server.HTTP.Port <= 0 { return errors.New("please provide the HTTP port in the convoy.json file") diff --git a/internal/pkg/fflag/fflag.go b/internal/pkg/fflag/fflag.go index e65a2111fb..1492fca812 100644 --- a/internal/pkg/fflag/fflag.go +++ b/internal/pkg/fflag/fflag.go @@ -9,7 +9,9 @@ import ( "text/tabwriter" ) -var ErrFeatureNotEnabled = errors.New("this feature is not enabled") +var ErrCircuitBreakerNotEnabled = errors.New("[feature flag] circuit breaker is not enabled") +var ErrFullTextSearchNotEnabled = errors.New("[feature flag] full text search is not enabled") +var ErrPrometheusMetricsNotEnabled = errors.New("[feature flag] prometheus metrics is not enabled") type ( FeatureFlagKey string @@ -18,6 +20,7 @@ type ( const ( Prometheus FeatureFlagKey = "prometheus" FullTextSearch FeatureFlagKey = "full-text-search" + CircuitBreaker FeatureFlagKey = "circuit-breaker" ) type ( @@ -32,25 +35,30 @@ const ( var DefaultFeaturesState = map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: disabled, + CircuitBreaker: disabled, } type FFlag struct { Features map[FeatureFlagKey]FeatureFlagState } -func NewFFlag(c *config.Configuration) (*FFlag, error) { +func NewFFlag(c *config.Configuration) *FFlag { f := &FFlag{ Features: clone(DefaultFeaturesState), } + for _, flag := range c.EnableFeatureFlag { switch flag { case string(Prometheus): f.Features[Prometheus] = enabled case string(FullTextSearch): f.Features[FullTextSearch] = enabled + case string(CircuitBreaker): + f.Features[CircuitBreaker] = enabled } } - return f, nil + + return f } func clone(src map[FeatureFlagKey]FeatureFlagState) map[FeatureFlagKey]FeatureFlagState { diff --git a/internal/pkg/fflag/fflag_test.go b/internal/pkg/fflag/fflag_test.go index 72fef28c6f..27d54e30b8 100644 --- a/internal/pkg/fflag/fflag_test.go +++ b/internal/pkg/fflag/fflag_test.go @@ -198,11 +198,8 @@ func TestNewFFlag(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewFFlag(tt.args.c) - if (err != nil) != tt.wantErr { - t.Errorf("NewFFlag() error = %v, wantErr %v", err, tt.wantErr) - return - } + got := NewFFlag(tt.args.c) + if !reflect.DeepEqual(got, tt.want) { t.Errorf("NewFFlag() got = %v, want %v", got, tt.want) } diff --git a/pkg/log/log.go b/pkg/log/log.go index dc64b8f34a..fb156bb618 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -95,7 +95,7 @@ func (l Level) ToLogrusLevel() (logrus.Level, error) { } // NewLogger creates and returns a new instance of Logger. -// Log level is set to DebugLevel by default. +// The log level is set to ErrorLevel by default. func NewLogger(out io.Writer) *Logger { log := &logrus.Logger{ Out: out, @@ -218,7 +218,7 @@ func (l *Logger) SetLevel(v Level) { l.logger.SetLevel(lvl) } -// WithField sets logger fields +// SetPrefix sets logger fields func (l *Logger) SetPrefix(value interface{}) { l.entry = l.entry.WithField("source", value) } diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index a9709f7c0e..0bc3cae0e6 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/circuit_breaker" @@ -34,7 +35,7 @@ import ( func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, licenser license.Licenser, projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, - attemptsRepo datastore.DeliveryAttemptsRepository, circuitBreakerManager *circuit_breaker.CircuitBreakerManager, + attemptsRepo datastore.DeliveryAttemptsRepository, circuitBreakerManager *circuit_breaker.CircuitBreakerManager, featureFlag *fflag.FFlag, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) (err error) { var data EventDelivery @@ -116,7 +117,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive return &RateLimitError{Err: ErrRateLimit, delay: time.Duration(endpoint.RateLimitDuration) * time.Second} } - if licenser.CircuitBreaking() { + if featureFlag.CanAccessFeature(fflag.CircuitBreaker) && licenser.CircuitBreaking() { breakerErr := circuitBreakerManager.CanExecute(ctx, endpoint.UID) if breakerErr != nil { return &CircuitBreakerError{Err: breakerErr} diff --git a/worker/task/process_retry_event_delivery.go b/worker/task/process_retry_event_delivery.go index 1dd8b76827..f606a91dd5 100644 --- a/worker/task/process_retry_event_delivery.go +++ b/worker/task/process_retry_event_delivery.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/pkg/circuit_breaker" "time" @@ -38,9 +40,9 @@ var ( defaultEventDelay = 120 * time.Second ) -func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, +func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, licenser license.Licenser, projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, - attemptsRepo datastore.DeliveryAttemptsRepository, manager *circuit_breaker.CircuitBreakerManager, + attemptsRepo datastore.DeliveryAttemptsRepository, circuitBreakerManager *circuit_breaker.CircuitBreakerManager, featureFlag *fflag.FFlag, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var data EventDelivery @@ -100,9 +102,28 @@ func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventD return &RateLimitError{Err: ErrRateLimit, delay: time.Duration(endpoint.RateLimitDuration) * time.Second} } - err = manager.CanExecute(ctx, endpoint.UID) - if err != nil { - return &DeliveryError{Err: err} + if featureFlag.CanAccessFeature(fflag.CircuitBreaker) && licenser.CircuitBreaking() { + breakerErr := circuitBreakerManager.CanExecute(ctx, endpoint.UID) + if breakerErr != nil { + return &CircuitBreakerError{Err: breakerErr} + } + + // check the circuit breaker state so we can disable the endpoint + cb, breakerErr := circuitBreakerManager.GetCircuitBreaker(ctx, endpoint.UID) + if breakerErr != nil { + return &CircuitBreakerError{Err: breakerErr} + } + + if cb != nil { + if cb.ConsecutiveFailures > circuitBreakerManager.GetConfig().ConsecutiveFailureThreshold { + endpointStatus := datastore.InactiveEndpointStatus + + breakerErr = endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) + if breakerErr != nil { + log.WithError(breakerErr).Error("failed to deactivate endpoint after failed retry") + } + } + } } err = eventDeliveryRepo.UpdateStatusOfEventDelivery(ctx, project.UID, *eventDelivery, datastore.ProcessingEventStatus) diff --git a/worker/task/search_tokenizer.go b/worker/task/search_tokenizer.go index 70f060a204..c14aff09b3 100644 --- a/worker/task/search_tokenizer.go +++ b/worker/task/search_tokenizer.go @@ -86,12 +86,11 @@ func tokenize(ctx context.Context, eventRepo datastore.EventRepository, jobRepo if err != nil { return err } - fflag, err := fflag2.NewFFlag(&cfg) - if err != nil { - return nil - } + + fflag := fflag2.NewFFlag(&cfg) + if !fflag.CanAccessFeature(fflag2.FullTextSearch) { - return fflag2.ErrFeatureNotEnabled + return fflag2.ErrFullTextSearchNotEnabled } // check if a job for a given project is currently running From 5eae27e33353c7fa9ae7827900456a628eee79e8 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 18 Sep 2024 10:27:55 +0200 Subject: [PATCH 26/48] chore: update tests --- cmd/agent/agent.go | 2 +- config/config_test.go | 12 ++++---- database/postgres/configuration_test.go | 2 +- internal/pkg/fflag/fflag_test.go | 9 ++++++ .../circuit_breaker_manager_test.go | 24 ++++++++-------- worker/task/process_event_delivery_test.go | 13 ++++++++- worker/task/process_retry_event_delivery.go | 28 +++++++++++-------- .../task/process_retry_event_delivery_test.go | 23 ++++++++++++++- 8 files changed, 80 insertions(+), 33 deletions(-) diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index 513b2675ca..8b53d3f629 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -168,7 +168,7 @@ func startServerComponent(_ context.Context, a *cli.App) error { srv.SetHandler(evHandler.BuildDataPlaneRoutes()) - log.Info("Started convoy server in %s\n", time.Since(start)) + log.Infof("Started convoy server in %s\n", time.Since(start)) httpConfig := cfg.Server.HTTP if httpConfig.SSL { diff --git a/config/config_test.go b/config/config_test.go index 145f57cc39..c0391c7287 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -128,11 +128,11 @@ func TestLoadConfig(t *testing.T) { CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 10, + FailureThreshold: 70, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{5, 10, 15}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Server: ServerConfiguration{ @@ -209,11 +209,11 @@ func TestLoadConfig(t *testing.T) { CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 10, + FailureThreshold: 70, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{5, 10, 15}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Redis: RedisConfiguration{ @@ -285,11 +285,11 @@ func TestLoadConfig(t *testing.T) { CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 10, + FailureThreshold: 70, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{5, 10, 15}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Database: DatabaseConfiguration{ diff --git a/database/postgres/configuration_test.go b/database/postgres/configuration_test.go index e51189ef14..a83ef554c9 100644 --- a/database/postgres/configuration_test.go +++ b/database/postgres/configuration_test.go @@ -116,7 +116,7 @@ func generateConfig() *datastore.Configuration { CircuitBreakerConfig: &datastore.CircuitBreakerConfig{ SampleRate: 30, ErrorTimeout: 30, - FailureThreshold: 0.1, + FailureThreshold: 10, FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, diff --git a/internal/pkg/fflag/fflag_test.go b/internal/pkg/fflag/fflag_test.go index 27d54e30b8..b41489f80b 100644 --- a/internal/pkg/fflag/fflag_test.go +++ b/internal/pkg/fflag/fflag_test.go @@ -27,6 +27,7 @@ func TestFFlag_CanAccessFeature(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: enabled, + CircuitBreaker: disabled, }, }, args: struct { @@ -44,6 +45,7 @@ func TestFFlag_CanAccessFeature(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: enabled, + CircuitBreaker: disabled, }, }, args: struct { @@ -61,6 +63,7 @@ func TestFFlag_CanAccessFeature(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: enabled, FullTextSearch: enabled, + CircuitBreaker: disabled, }, }, args: struct { @@ -78,6 +81,7 @@ func TestFFlag_CanAccessFeature(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: enabled, FullTextSearch: enabled, + CircuitBreaker: disabled, }, }, args: struct { @@ -95,6 +99,7 @@ func TestFFlag_CanAccessFeature(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: disabled, + CircuitBreaker: disabled, }, }, args: struct { @@ -112,6 +117,7 @@ func TestFFlag_CanAccessFeature(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: disabled, + CircuitBreaker: disabled, }, }, args: struct { @@ -163,6 +169,7 @@ func TestNewFFlag(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: disabled, + CircuitBreaker: disabled, }, }, wantErr: false, @@ -178,6 +185,7 @@ func TestNewFFlag(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: enabled, FullTextSearch: disabled, + CircuitBreaker: disabled, }, }, wantErr: false, @@ -191,6 +199,7 @@ func TestNewFFlag(t *testing.T) { Features: map[FeatureFlagKey]FeatureFlagState{ Prometheus: disabled, FullTextSearch: disabled, + CircuitBreaker: disabled, }, }, wantErr: false, diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index a65fababa0..b56064d312 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -53,11 +53,11 @@ func TestCircuitBreakerManager(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, BreakerTimeout: 30, - FailureThreshold: 10, + FailureThreshold: 70, FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } @@ -100,17 +100,19 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { keys, err := re.Keys(ctx, "breaker*").Result() require.NoError(t, err) - err = re.Del(ctx, keys...).Err() - require.NoError(t, err) + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } c := &CircuitBreakerConfig{ SampleRate: 2, BreakerTimeout: 30, - FailureThreshold: 10, + FailureThreshold: 70, FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -165,7 +167,7 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { FailureCount: 3, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -231,11 +233,11 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, BreakerTimeout: 30, - FailureThreshold: 50, + FailureThreshold: 70, FailureCount: 3, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 3, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -284,11 +286,11 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, BreakerTimeout: 30, - FailureThreshold: 50, + FailureThreshold: 70, FailureCount: 3, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10}, + NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 59c88533ca..bd94fa3a30 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -3,6 +3,7 @@ package task import ( "context" "encoding/json" + "github.com/frain-dev/convoy/internal/pkg/fflag" "testing" "github.com/frain-dev/convoy/internal/pkg/license" @@ -260,6 +261,8 @@ func TestProcessEventDelivery(t *testing.T) { licenser, _ := l.(*mocks.MockLicenser) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) + + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) }, nFn: func() func() { httpmock.Activate() @@ -348,6 +351,7 @@ func TestProcessEventDelivery(t *testing.T) { licenser, _ := l.(*mocks.MockLicenser) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) }, nFn: func() func() { httpmock.Activate() @@ -435,6 +439,7 @@ func TestProcessEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { @@ -525,6 +530,7 @@ func TestProcessEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { @@ -612,6 +618,7 @@ func TestProcessEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { @@ -699,6 +706,7 @@ func TestProcessEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) }, nFn: func() func() { @@ -791,6 +799,7 @@ func TestProcessEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { @@ -884,6 +893,7 @@ func TestProcessEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { @@ -965,7 +975,8 @@ func TestProcessEventDelivery(t *testing.T) { ) require.NoError(t, err) - processFn := ProcessEventDelivery(endpointRepo, msgRepo, licenser, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) + featureFlag := fflag.NewFFlag(&cfg) + processFn := ProcessEventDelivery(endpointRepo, msgRepo, licenser, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager, featureFlag) payload := EventDelivery{ EventDeliveryID: tc.msg.UID, diff --git a/worker/task/process_retry_event_delivery.go b/worker/task/process_retry_event_delivery.go index f606a91dd5..ae1ad7fe91 100644 --- a/worker/task/process_retry_event_delivery.go +++ b/worker/task/process_retry_event_delivery.go @@ -176,7 +176,7 @@ func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventD } var httpDuration time.Duration - if endpoint.HttpTimeout == 0 { + if endpoint.HttpTimeout == 0 || !licenser.AdvancedEndpointMgmt() { httpDuration = convoy.HTTP_TIMEOUT_IN_DURATION } else { httpDuration = time.Duration(endpoint.HttpTimeout) * time.Second @@ -226,21 +226,23 @@ func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventD log.Errorf("%s failed. Reason: %s", eventDelivery.UID, err) } - if done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint { + if done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint && !licenser.CircuitBreaking() { endpointStatus := datastore.ActiveEndpointStatus err := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) if err != nil { log.WithError(err).Error("Failed to reactivate endpoint after successful retry") } - // send endpoint reactivation notification - err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, false, resp.Error, string(resp.Body), resp.StatusCode) - if err != nil { - log.FromContext(ctx).WithError(err).Error("failed to send notification") + if licenser.AdvancedEndpointMgmt() { + // send endpoint reactivation notification + err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, false, resp.Error, string(resp.Body), resp.StatusCode) + if err != nil { + log.FromContext(ctx).WithError(err).Error("failed to send notification") + } } } - if !done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint { + if !done && endpoint.Status == datastore.PendingEndpointStatus && project.Config.DisableEndpoint && !licenser.CircuitBreaking() { endpointStatus := datastore.InactiveEndpointStatus err := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) if err != nil { @@ -264,7 +266,7 @@ func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventD eventDelivery.Status = datastore.FailureEventStatus } - if endpoint.Status != datastore.PendingEndpointStatus && project.Config.DisableEndpoint { + if endpoint.Status != datastore.PendingEndpointStatus && project.Config.DisableEndpoint && !licenser.CircuitBreaking() { endpointStatus := datastore.InactiveEndpointStatus err := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) @@ -272,10 +274,12 @@ func ProcessRetryEventDelivery(endpointRepo datastore.EndpointRepository, eventD log.WithError(err).Error("failed to deactivate endpoint after failed retry") } - // send endpoint deactivation notification - err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, true, resp.Error, string(resp.Body), resp.StatusCode) - if err != nil { - log.WithError(err).Error("failed to send notification") + if licenser.AdvancedEndpointMgmt() { + // send endpoint deactivation notification + err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, true, resp.Error, string(resp.Body), resp.StatusCode) + if err != nil { + log.WithError(err).Error("failed to send notification") + } } } } diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 8003e95565..49269e277c 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/frain-dev/convoy/internal/pkg/fflag" "testing" "github.com/frain-dev/convoy/internal/pkg/license" @@ -266,6 +267,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -354,6 +357,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -442,6 +447,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -532,6 +539,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -622,6 +631,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -712,6 +723,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(false) }, nFn: func() func() { @@ -799,6 +812,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -891,6 +906,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -984,6 +1001,8 @@ func TestProcessRetryEventDelivery(t *testing.T) { Return(nil).Times(1) licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().CircuitBreaking().Times(1).Return(false) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { @@ -1063,7 +1082,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { ) require.NoError(t, err) - processFn := ProcessRetryEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager) + featureFlag := fflag.NewFFlag(&cfg) + + processFn := ProcessRetryEventDelivery(endpointRepo, msgRepo, licenser, projectRepo, q, rateLimiter, dispatcher, attemptsRepo, manager, featureFlag) payload := EventDelivery{ EventDeliveryID: tc.msg.UID, From ed42554a2361d124a8de24b3ec95da4dcef6f033 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 18 Sep 2024 10:32:07 +0200 Subject: [PATCH 27/48] chore: fix go lint --- pkg/circuit_breaker/config.go | 4 ++-- pkg/circuit_breaker/config_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go index f7f58a428f..13974c8c97 100644 --- a/pkg/circuit_breaker/config.go +++ b/pkg/circuit_breaker/config.go @@ -51,8 +51,8 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - if c.FailureThreshold < 0 || c.FailureThreshold > 100 { - errs.WriteString("FailureThreshold must be between 0 and 100") + if c.FailureThreshold == 0 || c.FailureThreshold > 100 { + errs.WriteString("FailureThreshold must be between 1 and 100") errs.WriteString("; ") } diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index caae00ef78..63c5fca9e9 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -51,7 +51,7 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { FailureThreshold: 150, }, wantErr: true, - err: "FailureThreshold must be between 0 and 100", + err: "FailureThreshold must be between 1 and 100", }, { name: "Invalid FailureCount", From 6ba935c2f6a822b1a1b45c74e347556b13431183 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 20 Sep 2024 10:47:33 +0200 Subject: [PATCH 28/48] feat: refactor the sample query to discard entries that occurred before the breaker was reset --- cmd/hooks/hooks.go | 1 + cmd/worker/worker.go | 7 +- config/config.go | 2 + database/postgres/configuration.go | 9 +- database/postgres/delivery_attempts.go | 34 +++++- datastore/models.go | 6 +- datastore/repository.go | 2 +- mocks/repository.go | 10 +- pkg/circuit_breaker/circuit_breaker.go | 21 +++- .../circuit_breaker_manager.go | 114 ++++++++++++------ .../circuit_breaker_manager_test.go | 113 +++++++++-------- pkg/circuit_breaker/circuit_breaker_test.go | 5 +- pkg/circuit_breaker/config.go | 16 ++- pkg/circuit_breaker/config_test.go | 21 +++- pkg/circuit_breaker/store.go | 14 +-- pkg/circuit_breaker/store_test.go | 5 +- sql/1724236900.sql | 11 +- 17 files changed, 261 insertions(+), 130 deletions(-) diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index 13a4104f5f..8fd4893834 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -330,6 +330,7 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat FailureThreshold: cfg.CircuitBreaker.FailureThreshold, SuccessThreshold: cfg.CircuitBreaker.SuccessThreshold, ObservabilityWindow: cfg.CircuitBreaker.ObservabilityWindow, + MinimumRequestCount: cfg.CircuitBreaker.MinimumRequestCount, NotificationThresholds: notificationThresholds, ConsecutiveFailureThreshold: cfg.CircuitBreaker.ConsecutiveFailureThreshold, } diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 6367963ac6..d5d77f088e 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -253,7 +253,12 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte circuitBreakerManager, err = cb.NewCircuitBreakerManager( cb.ConfigOption(configuration.ToCircuitBreakerConfig()), cb.StoreOption(cb.NewRedisStore(rd.Client(), clock.NewRealClock())), - cb.ClockOption(clock.NewRealClock())) + cb.ClockOption(clock.NewRealClock()), + cb.NotificationFunctionOption(func(c cb.CircuitBreaker) error { + fmt.Printf("notification: %+v\n", c) + return nil + }), + ) if err != nil { a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") } diff --git a/config/config.go b/config/config.go index cd1a25c259..bcb35f816b 100644 --- a/config/config.go +++ b/config/config.go @@ -81,6 +81,7 @@ var DefaultConfiguration = Configuration{ FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, @@ -277,6 +278,7 @@ type CircuitBreakerConfiguration struct { ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` FailureThreshold uint64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` + MinimumRequestCount uint64 `json:"minimum_request_count" envconfig:"CONVOY_MINIMUM_REQUEST_COUNT"` ObservabilityWindow uint64 `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` NotificationThresholds [3]uint64 `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` diff --git a/database/postgres/configuration.go b/database/postgres/configuration.go index 8579c10d28..845163ff21 100644 --- a/database/postgres/configuration.go +++ b/database/postgres/configuration.go @@ -23,9 +23,10 @@ const ( cb_sample_rate,cb_error_timeout, cb_failure_threshold,cb_failure_count, cb_success_threshold,cb_observability_window, - cb_notification_thresholds,cb_consecutive_failure_threshold + cb_notification_thresholds,cb_consecutive_failure_threshold, + cb_minimum_request_count ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22); + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23); ` fetchConfiguration = ` @@ -50,6 +51,7 @@ const ( cb_failure_count AS "breaker_config.failure_count", cb_success_threshold AS "breaker_config.success_threshold", cb_observability_window AS "breaker_config.observability_window", + cb_minimum_request_count as "breaker_config.minimum_request_count", cb_notification_thresholds::INTEGER[] AS "breaker_config.notification_thresholds", cb_consecutive_failure_threshold AS "breaker_config.consecutive_failure_threshold", created_at, @@ -84,6 +86,7 @@ const ( cb_observability_window = $20, cb_notification_thresholds = $21, cb_consecutive_failure_threshold = $22, + cb_minimum_request_count = $23, updated_at = NOW() WHERE id = $1 AND deleted_at IS NULL; ` @@ -140,6 +143,7 @@ func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore. cb.ObservabilityWindow, cb.NotificationThresholds, cb.ConsecutiveFailureThreshold, + cb.MinimumRequestCount, ) if err != nil { return err @@ -213,6 +217,7 @@ func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Con cb.ObservabilityWindow, cb.NotificationThresholds, cb.ConsecutiveFailureThreshold, + cb.MinimumRequestCount, ) if err != nil { return err diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index 0fd219878c..bfb37634f7 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/circuit_breaker" @@ -131,7 +132,10 @@ func (d *deliveryAttemptRepo) DeleteProjectDeliveriesAttempts(ctx context.Contex return nil } -func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64) (results []circuit_breaker.PollResult, err error) { +func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64, resetTimes map[string]time.Time) (map[string]circuit_breaker.PollResult, error) { + resultsMap := map[string]circuit_breaker.PollResult{} + fmt.Printf("rowValue: %+v\n", resetTimes) + query := ` SELECT endpoint_id AS key, @@ -152,10 +156,34 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo if rowScanErr := rows.StructScan(&rowValue); rowScanErr != nil { return nil, rowScanErr } - results = append(results, rowValue) + resultsMap[rowValue.Key] = rowValue + } + + // this is an n+1 query? yikes + query2 := ` + SELECT + endpoint_id AS key, + COUNT(CASE WHEN status = false THEN 1 END) AS failures, + COUNT(CASE WHEN status = true THEN 1 END) AS successes + FROM convoy.delivery_attempts + WHERE endpoint_id = $1 AND created_at >= $2 group by endpoint_id; + ` + + for k, t := range resetTimes { + var rowValue circuit_breaker.PollResult + err = d.db.QueryRowxContext(ctx, query2, k, t).StructScan(&rowValue) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + } + + if &rowValue != nil { + resultsMap[k] = rowValue + } } - return results, nil + return resultsMap, nil } func (d *deliveryAttemptRepo) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { diff --git a/datastore/models.go b/datastore/models.go index d537482ee2..120335d49c 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -1352,11 +1352,12 @@ func (c *Configuration) ToCircuitBreakerConfig() *circuit_breaker.CircuitBreaker return &circuit_breaker.CircuitBreakerConfig{ SampleRate: c.CircuitBreakerConfig.SampleRate, + FailureCount: c.CircuitBreakerConfig.FailureCount, BreakerTimeout: c.CircuitBreakerConfig.ErrorTimeout, FailureThreshold: c.CircuitBreakerConfig.FailureThreshold, - FailureCount: c.CircuitBreakerConfig.FailureCount, SuccessThreshold: c.CircuitBreakerConfig.SuccessThreshold, ObservabilityWindow: c.CircuitBreakerConfig.ObservabilityWindow, + MinimumRequestCount: c.CircuitBreakerConfig.MinimumRequestCount, NotificationThresholds: notificationThresholds, ConsecutiveFailureThreshold: c.CircuitBreakerConfig.ConsecutiveFailureThreshold, } @@ -1392,10 +1393,11 @@ type OnPremStorage struct { type CircuitBreakerConfig struct { SampleRate uint64 `json:"sample_rate" db:"sample_rate"` ErrorTimeout uint64 `json:"error_timeout" db:"error_timeout"` - FailureThreshold uint64 `json:"failure_threshold" db:"failure_threshold"` FailureCount uint64 `json:"failure_count" db:"failure_count"` + FailureThreshold uint64 `json:"failure_threshold" db:"failure_threshold"` SuccessThreshold uint64 `json:"success_threshold" db:"success_threshold"` ObservabilityWindow uint64 `json:"observability_window" db:"observability_window"` + MinimumRequestCount uint64 `json:"minimum_request_count" db:"minimum_request_count"` NotificationThresholds pq.Int64Array `json:"notification_thresholds" db:"notification_thresholds"` ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" db:"consecutive_failure_threshold"` } diff --git a/datastore/repository.go b/datastore/repository.go index 13be810090..09420de53d 100644 --- a/datastore/repository.go +++ b/datastore/repository.go @@ -206,5 +206,5 @@ type DeliveryAttemptsRepository interface { FindDeliveryAttemptById(context.Context, string, string) (*DeliveryAttempt, error) FindDeliveryAttempts(context.Context, string) ([]DeliveryAttempt, error) DeleteProjectDeliveriesAttempts(ctx context.Context, projectID string, filter *DeliveryAttemptsFilter, hardDelete bool) error - GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64) (results []circuit_breaker.PollResult, err error) + GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64, resetTimes map[string]time.Time) (resultsMap map[string]circuit_breaker.PollResult, err error) } diff --git a/mocks/repository.go b/mocks/repository.go index 6351bd79f4..81295a8b60 100644 --- a/mocks/repository.go +++ b/mocks/repository.go @@ -2583,16 +2583,16 @@ func (mr *MockDeliveryAttemptsRepositoryMockRecorder) FindDeliveryAttempts(arg0, } // GetFailureAndSuccessCounts mocks base method. -func (m *MockDeliveryAttemptsRepository) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64) ([]circuit_breaker.PollResult, error) { +func (m *MockDeliveryAttemptsRepository) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64, resetTimes map[string]time.Time) (map[string]circuit_breaker.PollResult, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetFailureAndSuccessCounts", ctx, lookBackDuration) - ret0, _ := ret[0].([]circuit_breaker.PollResult) + ret := m.ctrl.Call(m, "GetFailureAndSuccessCounts", ctx, lookBackDuration, resetTimes) + ret0, _ := ret[0].(map[string]circuit_breaker.PollResult) ret1, _ := ret[1].(error) return ret0, ret1 } // GetFailureAndSuccessCounts indicates an expected call of GetFailureAndSuccessCounts. -func (mr *MockDeliveryAttemptsRepositoryMockRecorder) GetFailureAndSuccessCounts(ctx, lookBackDuration any) *gomock.Call { +func (mr *MockDeliveryAttemptsRepositoryMockRecorder) GetFailureAndSuccessCounts(ctx, lookBackDuration, resetTimes any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFailureAndSuccessCounts", reflect.TypeOf((*MockDeliveryAttemptsRepository)(nil).GetFailureAndSuccessCounts), ctx, lookBackDuration) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFailureAndSuccessCounts", reflect.TypeOf((*MockDeliveryAttemptsRepository)(nil).GetFailureAndSuccessCounts), ctx, lookBackDuration, resetTimes) } diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 34059e5a4d..5d8cbd9669 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -1,6 +1,7 @@ package circuit_breaker import ( + "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/pkg/msgpack" "time" ) @@ -15,7 +16,7 @@ type CircuitBreaker struct { Requests uint64 `json:"requests"` // Percentage of failures in the observability window FailureRate float64 `json:"failure_rate"` - // Time after which the circuit breaker will reset + // Time after which the circuit breaker will reset when in half-open state WillResetAt time.Time `json:"will_reset_at"` // Number of failed requests in the observability window TotalFailures uint64 `json:"total_failures"` @@ -23,15 +24,26 @@ type CircuitBreaker struct { TotalSuccesses uint64 `json:"total_successes"` // Number of consecutive circuit breaker trips ConsecutiveFailures uint64 `json:"consecutive_failures"` + // Number of notifications (maximum of 3) sent in the observability window + NotificationsSent uint64 `json:"notifications_sent"` } -func (b *CircuitBreaker) String() (s string, err error) { +func NewCircuitBreaker(key string) *CircuitBreaker { + return &CircuitBreaker{ + Key: key, + State: StateClosed, + NotificationsSent: 0, + } +} + +func (b *CircuitBreaker) String() (s string) { bytes, err := msgpack.EncodeMsgPack(b) if err != nil { - return "", err + log.WithError(err).Error("[circuit breaker] failed to encode circuit breaker") + return "" } - return string(bytes), nil + return string(bytes) } func (b *CircuitBreaker) tripCircuitBreaker(resetTime time.Time) { @@ -46,5 +58,6 @@ func (b *CircuitBreaker) toHalfOpen() { func (b *CircuitBreaker) resetCircuitBreaker() { b.State = StateClosed + b.NotificationsSent = 0 b.ConsecutiveFailures = 0 } diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index b86c4ca54e..39e1825c33 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -7,18 +7,19 @@ import ( "github.com/frain-dev/convoy/pkg/clock" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/pkg/msgpack" + "strings" "time" ) -// todo(raymond): add to feature flags -// todo(raymond): notification thresholds are percentages +// todo(raymond): send notifications when notification thresholds are hit, then update breaker state +// todo(raymond): save the previous failure rate along side the current so we can compare to see if it's reducing // todo(raymond): metrics should contain error rate // todo(raymond): use a guage for failure rate metrics const prefix = "breaker:" const mutexKey = "convoy:circuit_breaker:mutex" -type PollFunc func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) +type PollFunc func(ctx context.Context, lookBackDuration uint64, resetTimes map[string]time.Time) (map[string]PollResult, error) type CircuitBreakerOption func(cb *CircuitBreakerManager) error var ( @@ -39,6 +40,9 @@ var ( // ErrConfigMustNotBeNil is returned when a nil config is passed to NewCircuitBreakerManager ErrConfigMustNotBeNil = errors.New("[circuit breaker] config must not be nil") + + // ErrNotificationFunctionMustNotBeNil is returned when a nil function is passed to NewCircuitBreakerManager + ErrNotificationFunctionMustNotBeNil = errors.New("[circuit breaker] notification function must not be nil") ) // State represents a state of a CircuitBreaker. @@ -71,9 +75,10 @@ type PollResult struct { } type CircuitBreakerManager struct { - config *CircuitBreakerConfig - clock clock.Clock - store CircuitBreakerStore + config *CircuitBreakerConfig + clock clock.Clock + store CircuitBreakerStore + notificationFn func(CircuitBreaker) error } func NewCircuitBreakerManager(options ...CircuitBreakerOption) (*CircuitBreakerManager, error) { @@ -98,6 +103,10 @@ func NewCircuitBreakerManager(options ...CircuitBreakerOption) (*CircuitBreakerM return nil, ErrConfigMustNotBeNil } + if r.notificationFn == nil { + return nil, ErrNotificationFunctionMustNotBeNil + } + return r, nil } @@ -138,15 +147,28 @@ func ConfigOption(config *CircuitBreakerConfig) CircuitBreakerOption { } } -func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults []PollResult) error { +func NotificationFunctionOption(fn func(c CircuitBreaker) error) CircuitBreakerOption { + return func(cb *CircuitBreakerManager) error { + if fn == nil { + return ErrNotificationFunctionMustNotBeNil + } + + cb.notificationFn = fn + return nil + } +} + +func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults map[string]PollResult) error { redisCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - keys := make([]string, len(pollResults)) - for i := range pollResults { - key := fmt.Sprintf("%s%s", prefix, pollResults[i].Key) - keys[i] = key - pollResults[i].Key = key + circuitBreakerMap := make(map[string]CircuitBreaker, len(pollResults)) + + keys, j := make([]string, len(pollResults)), 0 + for k := range pollResults { + key := fmt.Sprintf("%s%s", prefix, k) + keys[j] = key + j++ } res, err := cb.store.GetMany(redisCtx, keys...) @@ -154,48 +176,34 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] return err } - circuitBreakers := make([]CircuitBreaker, len(pollResults)) for i := range res { if res[i] == nil { - c := CircuitBreaker{ - State: StateClosed, - Key: pollResults[i].Key, - } - circuitBreakers[i] = c + circuitBreakerMap[keys[i]] = *NewCircuitBreaker(keys[i]) continue } - c := CircuitBreaker{} str, ok := res[i].(string) if !ok { log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) // the circuit breaker is corrupted, create a new one in its place - circuitBreakers[i] = CircuitBreaker{ - State: StateClosed, - Key: keys[i], - } + circuitBreakerMap[keys[i]] = *NewCircuitBreaker(keys[i]) continue } + var c CircuitBreaker asBytes := []byte(str) innerErr := msgpack.DecodeMsgPack(asBytes, &c) if innerErr != nil { return innerErr } - circuitBreakers[i] = c + circuitBreakerMap[keys[i]] = c } - resultsMap := make(map[string]PollResult) - for _, result := range pollResults { - resultsMap[result.Key] = result - } - - circuitBreakerMap := make(map[string]CircuitBreaker, len(resultsMap)) - - for _, breaker := range circuitBreakers { - result := resultsMap[breaker.Key] + for key, breaker := range circuitBreakerMap { + k := strings.Split(key, ":") + result := pollResults[k[1]] breaker.TotalFailures = result.Failures breaker.TotalSuccesses = result.Successes @@ -209,16 +217,29 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults [] if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { breaker.resetCircuitBreaker() - } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && - (breaker.FailureRate >= float64(cb.config.FailureThreshold) || breaker.TotalFailures >= cb.config.FailureCount) { - breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) + } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && breaker.Requests >= cb.config.MinimumRequestCount { + if breaker.FailureRate >= float64(cb.config.FailureThreshold) || breaker.TotalFailures >= cb.config.FailureCount { + breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) + } } if breaker.State == StateOpen && cb.clock.Now().After(breaker.WillResetAt) { breaker.toHalfOpen() } - circuitBreakerMap[breaker.Key] = breaker + // send notifications for each circuit breaker + for i := range cb.config.NotificationThresholds { + if breaker.NotificationsSent < 3 && breaker.FailureRate >= float64(cb.config.NotificationThresholds[i]) { + innerErr := cb.notificationFn(breaker) + if innerErr != nil { + log.WithError(innerErr).Errorf("[circuit breaker] failed to execute notification function") + } + breaker.NotificationsSent++ + break + } + } + + circuitBreakerMap[key] = breaker } if err = cb.updateCircuitBreakers(ctx, circuitBreakerMap); err != nil { @@ -244,6 +265,10 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir return nil, err } + if len(keys) == 0 { + return []CircuitBreaker{}, nil + } + redisCtx2, cancel2 := context.WithTimeout(ctx, 5*time.Second) defer cancel2() @@ -367,8 +392,21 @@ func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc P } }() + bs, err := cb.loadCircuitBreakers(ctx) + if err != nil { + log.WithError(err).Error("[circuit breaker] failed to load circuitBreakers") + return err + } + + resetMap := make(map[string]time.Time, len(bs)) + for i := range bs { + if bs[i].State == StateClosed && bs[i].WillResetAt.After(time.Time{}) { + resetMap[bs[i].Key] = bs[i].WillResetAt + } + } + // Get the failure and success counts from the last X minutes - pollResults, err := pollFunc(ctx, cb.config.ObservabilityWindow) + pollResults, err := pollFunc(ctx, cb.config.ObservabilityWindow, resetMap) if err != nil { return fmt.Errorf("poll function failed: %w", err) } diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index b56064d312..2412dad23c 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -22,13 +22,15 @@ func getRedis(t *testing.T) (client redis.UniversalClient, err error) { return redis.NewClient(opts), nil } -func pollResult(t *testing.T, key string, failureCount, successCount uint64) PollResult { +func pollResult(t *testing.T, key string, failureCount, successCount uint64) map[string]PollResult { t.Helper() - return PollResult{ - Key: key, - Failures: failureCount, - Successes: successCount, + return map[string]PollResult{ + key: { + Key: key, + Failures: failureCount, + Successes: successCount, + }, } } @@ -56,22 +58,31 @@ func TestCircuitBreakerManager(t *testing.T) { FailureThreshold: 70, FailureCount: 3, SuccessThreshold: 1, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } - b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + b, err := NewCircuitBreakerManager( + ClockOption(testClock), + StoreOption(store), + ConfigOption(c), + NotificationFunctionOption(func(c CircuitBreaker) error { + t.Logf("c: %+v, s: %+v f: %+v\n", int64(c.FailureRate), c.State, c.NotificationsSent) + return nil + }), + ) require.NoError(t, err) endpointId := "endpoint-1" - pollResults := [][]PollResult{ - {pollResult(t, endpointId, 1, 0)}, - {pollResult(t, endpointId, 2, 0)}, - {pollResult(t, endpointId, 2, 1)}, - {pollResult(t, endpointId, 2, 2)}, - {pollResult(t, endpointId, 2, 3)}, - {pollResult(t, endpointId, 1, 4)}, + pollResults := []map[string]PollResult{ + pollResult(t, endpointId, 1, 0), + pollResult(t, endpointId, 2, 0), + pollResult(t, endpointId, 2, 1), + pollResult(t, endpointId, 2, 2), + pollResult(t, endpointId, 2, 3), + pollResult(t, endpointId, 1, 4), } for i := 0; i < len(pollResults); i++ { @@ -120,13 +131,13 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { endpoint1 := "endpoint-1" endpoint2 := "endpoint-2" - pollResults := [][]PollResult{ - {pollResult(t, endpoint1, 1, 0)}, - {pollResult(t, endpoint1, 2, 0)}, - {pollResult(t, endpoint1, 2, 1), pollResult(t, endpoint2, 1, 0)}, - {pollResult(t, endpoint1, 2, 2), pollResult(t, endpoint2, 1, 1)}, - {pollResult(t, endpoint1, 2, 3), pollResult(t, endpoint2, 0, 2)}, - {pollResult(t, endpoint1, 1, 4), pollResult(t, endpoint2, 1, 1)}, + pollResults := []map[string]PollResult{ + pollResult(t, endpoint1, 1, 0), + pollResult(t, endpoint1, 2, 0), + pollResult(t, endpoint1, 2, 1), pollResult(t, endpoint2, 1, 0), + pollResult(t, endpoint1, 2, 2), pollResult(t, endpoint2, 1, 1), + pollResult(t, endpoint1, 2, 3), pollResult(t, endpoint2, 0, 2), + pollResult(t, endpoint1, 1, 4), pollResult(t, endpoint2, 1, 1), } for i := 0; i < len(pollResults); i++ { @@ -174,14 +185,14 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { require.NoError(t, err) endpointId := "endpoint-1" - pollResults := [][]PollResult{ - {pollResult(t, endpointId, 1, 2)}, // Closed - {pollResult(t, endpointId, 3, 1)}, // Open (FailureCount reached) - {pollResult(t, endpointId, 0, 0)}, // Still Open - {pollResult(t, endpointId, 1, 1)}, // Half-Open (after ErrorTimeout) - {pollResult(t, endpointId, 0, 1)}, // Still Half-Open - {pollResult(t, endpointId, 0, 2)}, // Closed (SuccessThreshold reached) - {pollResult(t, endpointId, 4, 0)}, // Open (FailureThreshold reached) + pollResults := []map[string]PollResult{ + pollResult(t, endpointId, 1, 2), // Closed + pollResult(t, endpointId, 3, 1), // Open (FailureCount reached) + pollResult(t, endpointId, 0, 0), // Still Open + pollResult(t, endpointId, 1, 1), // Half-Open (after ErrorTimeout) + pollResult(t, endpointId, 0, 1), // Still Half-Open + pollResult(t, endpointId, 0, 2), // Closed (SuccessThreshold reached) + pollResult(t, endpointId, 4, 0), // Open (FailureThreshold reached) } expectedStates := []State{ @@ -244,12 +255,12 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { require.NoError(t, err) endpointId := "endpoint-1" - pollResults := [][]PollResult{ - {pollResult(t, endpointId, 3, 1)}, // Open - {pollResult(t, endpointId, 1, 1)}, // Half-Open - {pollResult(t, endpointId, 3, 0)}, // Open - {pollResult(t, endpointId, 1, 1)}, // Half-Open - {pollResult(t, endpointId, 3, 0)}, // Open + pollResults := []map[string]PollResult{ + pollResult(t, endpointId, 3, 1), // Open + pollResult(t, endpointId, 1, 1), // Half-Open + pollResult(t, endpointId, 3, 0), // Open + pollResult(t, endpointId, 1, 1), // Half-Open + pollResult(t, endpointId, 3, 0), // Open } for _, result := range pollResults { @@ -300,10 +311,10 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { endpoint2 := "endpoint-2" endpoint3 := "endpoint-3" - pollResults := [][]PollResult{ - {pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4)}, - {pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4)}, - {pollResult(t, endpoint1, 3, 1), pollResult(t, endpoint2, 1, 3), pollResult(t, endpoint3, 1, 5)}, + pollResults := []map[string]PollResult{ + pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4), + pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4), + pollResult(t, endpoint1, 3, 1), pollResult(t, endpoint2, 1, 3), pollResult(t, endpoint3, 1, 5), } for _, results := range pollResults { @@ -446,9 +457,9 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { require.NoError(t, err) ctx := context.Background() - pollResults := []PollResult{ - {Key: "test1", Failures: 3, Successes: 7}, - {Key: "test2", Failures: 6, Successes: 4}, + pollResults := map[string]PollResult{ + "test1": {Key: "test1", Failures: 3, Successes: 7}, + "test2": {Key: "test2", Failures: 6, Successes: 4}, } err = manager.sampleStore(ctx, pollResults) @@ -737,10 +748,10 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { ctx := context.Background() t.Run("Sample and Update Success", func(t *testing.T) { - pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { - return []PollResult{ - {Key: "test1", Failures: 3, Successes: 7}, - {Key: "test2", Failures: 6, Successes: 4}, + pollFunc := func(ctx context.Context, lookBackDuration uint64, _ map[string]time.Time) (map[string]PollResult, error) { + return map[string]PollResult{ + "test1": {Key: "test1", Failures: 3, Successes: 7}, + "test2": {Key: "test2", Failures: 6, Successes: 4}, }, nil } @@ -765,8 +776,8 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { t.Run("Sample and Update with Empty Results", func(t *testing.T) { - pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { - return []PollResult{}, nil + pollFunc := func(ctx context.Context, lookBackDuration uint64, _ map[string]time.Time) (map[string]PollResult, error) { + return map[string]PollResult{}, nil } err := manager.sampleAndUpdate(ctx, pollFunc) @@ -774,7 +785,7 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { }) t.Run("Sample and Update with Poll Function Error", func(t *testing.T) { - pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { + pollFunc := func(ctx context.Context, lookBackDuration uint64, _ map[string]time.Time) (map[string]PollResult, error) { return nil, errors.New("poll function error") } @@ -809,10 +820,10 @@ func TestCircuitBreakerManager_Start(t *testing.T) { defer cancel() pollCount := 0 - pollFunc := func(ctx context.Context, lookBackDuration uint64) ([]PollResult, error) { + pollFunc := func(ctx context.Context, lookBackDuration uint64, _ map[string]time.Time) (map[string]PollResult, error) { pollCount++ - return []PollResult{ - {Key: "test", Failures: uint64(pollCount), Successes: 10 - uint64(pollCount)}, + return map[string]PollResult{ + "test": {Key: "test", Failures: uint64(pollCount), Successes: 10 - uint64(pollCount)}, }, nil } diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index a93d59f11b..c206ddbc40 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -20,14 +20,13 @@ func TestCircuitBreaker_String(t *testing.T) { } t.Run("Success", func(t *testing.T) { - result, err := cb.String() + result := cb.String() - require.NoError(t, err) require.NotEmpty(t, result) // Decode the result back to a CircuitBreaker var decodedCB CircuitBreaker - err = msgpack.DecodeMsgPack([]byte(result), &decodedCB) + err := msgpack.DecodeMsgPack([]byte(result), &decodedCB) require.NoError(t, err) // Compare the decoded CircuitBreaker with the original diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go index 13974c8c97..34dd355f15 100644 --- a/pkg/circuit_breaker/config.go +++ b/pkg/circuit_breaker/config.go @@ -16,10 +16,15 @@ type CircuitBreakerConfig struct { BreakerTimeout uint64 `json:"breaker_timeout"` // FailureThreshold is the % of failed requests in the observability window - // after which the breaker will go into the open state + // after which a circuit breaker will go into the open state FailureThreshold uint64 `json:"failure_threshold"` + // MinimumRequestCount minimum number of requests in the observability window + // that will trip a circuit breaker + MinimumRequestCount uint64 `json:"request_count"` + // FailureCount total number of failed requests in the observability window + // that will trip a circuit breaker FailureCount uint64 `json:"failure_count"` // SuccessThreshold is the % of successful requests in the observability window @@ -30,7 +35,7 @@ type CircuitBreakerConfig struct { // polled when determining the number successful and failed requests ObservabilityWindow uint64 `json:"observability_window"` - // NotificationThresholds These are the error counts after which we will send out notifications. + // NotificationThresholds These are the error thresholds after which we will send out notifications. NotificationThresholds [3]uint64 `json:"notification_thresholds"` // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. @@ -56,6 +61,13 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } + fmt.Printf("%+v\n", c.MinimumRequestCount) + + if c.MinimumRequestCount < 10 { + errs.WriteString("MinimumRequestCount must be greater than 10") + errs.WriteString("; ") + } + if c.FailureCount == 0 { errs.WriteString("FailureCount must be greater than 0") errs.WriteString("; ") diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index 63c5fca9e9..cad5da5230 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -17,12 +17,13 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { config: CircuitBreakerConfig{ SampleRate: 1, BreakerTimeout: 30, - FailureThreshold: 5, + FailureThreshold: 50, FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, + MinimumRequestCount: 10, }, wantErr: false, }, @@ -148,6 +149,22 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { wantErr: true, err: "Notification threshold at index [2] = 60 must be less than the failure threshold", }, + { + name: "Invalid MinimumRequestCount", + config: CircuitBreakerConfig{ + SampleRate: 1, + FailureCount: 5, + BreakerTimeout: 30, + FailureThreshold: 30, + SuccessThreshold: 2, + ObservabilityWindow: 5, + MinimumRequestCount: 5, + NotificationThresholds: [3]uint64{30, 50, 60}, + ConsecutiveFailureThreshold: 1, + }, + wantErr: true, + err: "MinimumRequestCount must be greater than 10", + }, } for _, tt := range tests { @@ -157,6 +174,8 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { require.Error(t, err) require.NotEmpty(t, tt.err) require.Contains(t, err.Error(), tt.err) + } else { + require.NoError(t, err) } }) } diff --git a/pkg/circuit_breaker/store.go b/pkg/circuit_breaker/store.go index fc468dcfec..ee806e095f 100644 --- a/pkg/circuit_breaker/store.go +++ b/pkg/circuit_breaker/store.go @@ -102,12 +102,8 @@ func (s *RedisStore) SetOne(ctx context.Context, key string, value interface{}, func (s *RedisStore) SetMany(ctx context.Context, breakers map[string]CircuitBreaker, ttl time.Duration) error { pipe := s.redis.TxPipeline() for key, breaker := range breakers { - val, innerErr := breaker.String() - if innerErr != nil { - return innerErr - } - - if innerErr = pipe.Set(ctx, key, val, ttl).Err(); innerErr != nil { + val := breaker.String() + if innerErr := pipe.Set(ctx, key, val, ttl).Err(); innerErr != nil { return innerErr } } @@ -163,9 +159,9 @@ func (t *TestStore) GetOne(_ context.Context, s string) (string, error) { return "", ErrCircuitBreakerNotFound } - vv, err := res.String() - if err != nil { - return "", err + vv := res.String() + if vv != "" { + return "", errors.New("an error occurred decoding the circuit breaker") } return vv, nil diff --git a/pkg/circuit_breaker/store_test.go b/pkg/circuit_breaker/store_test.go index c3a606a03f..701aed6209 100644 --- a/pkg/circuit_breaker/store_test.go +++ b/pkg/circuit_breaker/store_test.go @@ -168,8 +168,7 @@ func TestRedisStore_SetMany(t *testing.T) { result, err := redisClient.Get(ctx, key).Result() require.NoError(t, err) - expectedValue, err := breaker.String() - require.NoError(t, err) + expectedValue := breaker.String() require.Equal(t, expectedValue, result) // Verify the expiration was set @@ -212,7 +211,7 @@ func TestTestStore_GetOne(t *testing.T) { result, err := store.GetOne(ctx, "test") require.NoError(t, err) - expectedValue, _ := cb.String() + expectedValue := cb.String() require.Equal(t, expectedValue, result) }) diff --git a/sql/1724236900.sql b/sql/1724236900.sql index a52669bee8..8f54fb2619 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -1,10 +1,11 @@ -- +migrate Up -alter table convoy.configurations add column if not exists cb_sample_rate int not null default 30; -alter table convoy.configurations add column if not exists cb_error_timeout int not null default 30; -alter table convoy.configurations add column if not exists cb_failure_threshold int not null default 70; +alter table convoy.configurations add column if not exists cb_sample_rate int not null default 30; -- seconds +alter table convoy.configurations add column if not exists cb_error_timeout int not null default 30; -- seconds +alter table convoy.configurations add column if not exists cb_failure_threshold int not null default 70; -- percentage alter table convoy.configurations add column if not exists cb_failure_count int not null default 1; -alter table convoy.configurations add column if not exists cb_success_threshold int not null default 5; -alter table convoy.configurations add column if not exists cb_observability_window int not null default 5; +alter table convoy.configurations add column if not exists cb_success_threshold int not null default 1; -- percentage +alter table convoy.configurations add column if not exists cb_observability_window int not null default 30; -- minutes +alter table convoy.configurations add column if not exists cb_minimum_request_count int not null default 10; alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[10, 30, 50]; alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 10; create index if not exists idx_delivery_attempts_created_at on convoy.delivery_attempts (created_at); From 29210893122e9767e72efeed062643269bfa98a1 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 24 Sep 2024 12:41:02 +0200 Subject: [PATCH 29/48] feat: make the success threshold a percentage; add success rate to the breaker; fixed tests; --- config/config_test.go | 3 + pkg/circuit_breaker/circuit_breaker.go | 2 + .../circuit_breaker_manager.go | 38 +++---- .../circuit_breaker_manager_test.go | 99 ++++++++++++------- pkg/circuit_breaker/config.go | 6 +- pkg/circuit_breaker/config_test.go | 4 +- pkg/circuit_breaker/store.go | 2 +- pkg/circuit_breaker/store_test.go | 28 ------ worker/task/process_event_delivery_test.go | 1 + .../task/process_retry_event_delivery_test.go | 1 + 10 files changed, 94 insertions(+), 90 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index c0391c7287..e714e1ece8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -132,6 +132,7 @@ func TestLoadConfig(t *testing.T) { FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, @@ -213,6 +214,7 @@ func TestLoadConfig(t *testing.T) { FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, @@ -289,6 +291,7 @@ func TestLoadConfig(t *testing.T) { FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 5d8cbd9669..1a48da233d 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -16,6 +16,8 @@ type CircuitBreaker struct { Requests uint64 `json:"requests"` // Percentage of failures in the observability window FailureRate float64 `json:"failure_rate"` + // Percentage of failures in the observability window + SuccessRate float64 `json:"success_rate"` // Time after which the circuit breaker will reset when in half-open state WillResetAt time.Time `json:"will_reset_at"` // Number of failed requests in the observability window diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 39e1825c33..3140f792aa 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -103,10 +103,6 @@ func NewCircuitBreakerManager(options ...CircuitBreakerOption) (*CircuitBreakerM return nil, ErrConfigMustNotBeNil } - if r.notificationFn == nil { - return nil, ErrNotificationFunctionMustNotBeNil - } - return r, nil } @@ -162,7 +158,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma redisCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - circuitBreakerMap := make(map[string]CircuitBreaker, len(pollResults)) + circuitBreakers := make(map[string]CircuitBreaker, len(pollResults)) keys, j := make([]string, len(pollResults)), 0 for k := range pollResults { @@ -178,7 +174,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma for i := range res { if res[i] == nil { - circuitBreakerMap[keys[i]] = *NewCircuitBreaker(keys[i]) + circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i]) continue } @@ -187,7 +183,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) // the circuit breaker is corrupted, create a new one in its place - circuitBreakerMap[keys[i]] = *NewCircuitBreaker(keys[i]) + circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i]) continue } @@ -198,10 +194,10 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma return innerErr } - circuitBreakerMap[keys[i]] = c + circuitBreakers[keys[i]] = c } - for key, breaker := range circuitBreakerMap { + for key, breaker := range circuitBreakers { k := strings.Split(key, ":") result := pollResults[k[1]] @@ -209,13 +205,16 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma breaker.TotalSuccesses = result.Successes breaker.Requests = breaker.TotalSuccesses + breaker.TotalFailures + prevFailureRate := breaker.FailureRate if breaker.Requests == 0 { breaker.FailureRate = 0 + breaker.SuccessRate = 0 } else { breaker.FailureRate = float64(breaker.TotalFailures) / float64(breaker.Requests) * 100 + breaker.SuccessRate = float64(breaker.TotalSuccesses) / float64(breaker.Requests) * 100 } - if breaker.State == StateHalfOpen && breaker.TotalSuccesses >= cb.config.SuccessThreshold { + if breaker.State == StateHalfOpen && breaker.SuccessRate >= float64(cb.config.SuccessThreshold) { breaker.resetCircuitBreaker() } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && breaker.Requests >= cb.config.MinimumRequestCount { if breaker.FailureRate >= float64(cb.config.FailureThreshold) || breaker.TotalFailures >= cb.config.FailureCount { @@ -228,21 +227,22 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma } // send notifications for each circuit breaker - for i := range cb.config.NotificationThresholds { - if breaker.NotificationsSent < 3 && breaker.FailureRate >= float64(cb.config.NotificationThresholds[i]) { - innerErr := cb.notificationFn(breaker) - if innerErr != nil { - log.WithError(innerErr).Errorf("[circuit breaker] failed to execute notification function") + if cb.notificationFn != nil { + if prevFailureRate < breaker.FailureRate && breaker.NotificationsSent < 3 { + if breaker.FailureRate >= float64(cb.config.NotificationThresholds[breaker.NotificationsSent]) { + innerErr := cb.notificationFn(breaker) + if innerErr != nil { + log.WithError(innerErr).Errorf("[circuit breaker] failed to execute notification function") + } + breaker.NotificationsSent++ } - breaker.NotificationsSent++ - break } } - circuitBreakerMap[key] = breaker + circuitBreakers[key] = breaker } - if err = cb.updateCircuitBreakers(ctx, circuitBreakerMap); err != nil { + if err = cb.updateCircuitBreakers(ctx, circuitBreakers); err != nil { log.WithError(err).Error("[circuit breaker] failed to update state") return err } diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index 2412dad23c..44c920ed6d 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -34,6 +34,12 @@ func pollResult(t *testing.T, key string, failureCount, successCount uint64) map } } +type notificationTriggered struct { + Rate float64 + Sent uint64 + State State +} + func TestCircuitBreakerManager(t *testing.T) { ctx := context.Background() @@ -57,19 +63,24 @@ func TestCircuitBreakerManager(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 70, FailureCount: 3, - SuccessThreshold: 1, + SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } + var triggered []notificationTriggered b, err := NewCircuitBreakerManager( ClockOption(testClock), StoreOption(store), ConfigOption(c), NotificationFunctionOption(func(c CircuitBreaker) error { - t.Logf("c: %+v, s: %+v f: %+v\n", int64(c.FailureRate), c.State, c.NotificationsSent) + triggered = append(triggered, notificationTriggered{ + State: c.State, + Rate: c.FailureRate, + Sent: c.NotificationsSent, + }) return nil }), ) @@ -77,10 +88,10 @@ func TestCircuitBreakerManager(t *testing.T) { endpointId := "endpoint-1" pollResults := []map[string]PollResult{ - pollResult(t, endpointId, 1, 0), - pollResult(t, endpointId, 2, 0), - pollResult(t, endpointId, 2, 1), - pollResult(t, endpointId, 2, 2), + pollResult(t, endpointId, 3, 9), + pollResult(t, endpointId, 5, 10), + pollResult(t, endpointId, 10, 1), + pollResult(t, endpointId, 20, 0), pollResult(t, endpointId, 2, 3), pollResult(t, endpointId, 1, 4), } @@ -92,6 +103,11 @@ func TestCircuitBreakerManager(t *testing.T) { testClock.AdvanceTime(time.Minute) } + require.Len(t, triggered, 3) + require.Equal(t, triggered[2].Sent, uint64(2)) + require.Equal(t, triggered[2].State, StateOpen) + require.Equal(t, int64(triggered[2].Rate), int64(90)) + breaker, innerErr := b.GetCircuitBreakerWithError(ctx, endpointId) require.NoError(t, innerErr) @@ -121,7 +137,8 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 70, FailureCount: 3, - SuccessThreshold: 1, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, @@ -176,7 +193,8 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 3, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, @@ -186,13 +204,12 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { endpointId := "endpoint-1" pollResults := []map[string]PollResult{ - pollResult(t, endpointId, 1, 2), // Closed - pollResult(t, endpointId, 3, 1), // Open (FailureCount reached) - pollResult(t, endpointId, 0, 0), // Still Open - pollResult(t, endpointId, 1, 1), // Half-Open (after ErrorTimeout) - pollResult(t, endpointId, 0, 1), // Still Half-Open - pollResult(t, endpointId, 0, 2), // Closed (SuccessThreshold reached) - pollResult(t, endpointId, 4, 0), // Open (FailureThreshold reached) + pollResult(t, endpointId, 1, 2), // Closed + pollResult(t, endpointId, 13, 1), // Open (FailureCount reached) + pollResult(t, endpointId, 13, 1), // Still Open + pollResult(t, endpointId, 10, 1), // Half-Open (after ErrorTimeout) + pollResult(t, endpointId, 0, 2), // Closed (SuccessThreshold reached) + pollResult(t, endpointId, 14, 0), // Open (FailureThreshold reached) } expectedStates := []State{ @@ -200,7 +217,6 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { StateOpen, StateOpen, StateHalfOpen, - StateHalfOpen, StateClosed, StateOpen, } @@ -246,7 +262,8 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 70, FailureCount: 3, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 3, @@ -256,11 +273,11 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { endpointId := "endpoint-1" pollResults := []map[string]PollResult{ - pollResult(t, endpointId, 3, 1), // Open - pollResult(t, endpointId, 1, 1), // Half-Open - pollResult(t, endpointId, 3, 0), // Open - pollResult(t, endpointId, 1, 1), // Half-Open - pollResult(t, endpointId, 3, 0), // Open + pollResult(t, endpointId, 13, 1), // Open + pollResult(t, endpointId, 13, 1), // Half-Open + pollResult(t, endpointId, 15, 0), // Open + pollResult(t, endpointId, 17, 1), // Half-Open + pollResult(t, endpointId, 13, 0), // Open } for _, result := range pollResults { @@ -297,10 +314,11 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { c := &CircuitBreakerConfig{ SampleRate: 2, BreakerTimeout: 30, - FailureThreshold: 70, + FailureThreshold: 60, FailureCount: 3, - SuccessThreshold: 2, + SuccessThreshold: 10, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } @@ -312,9 +330,9 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { endpoint3 := "endpoint-3" pollResults := []map[string]PollResult{ - pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4), - pollResult(t, endpoint1, 1, 1), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4), - pollResult(t, endpoint1, 3, 1), pollResult(t, endpoint2, 1, 3), pollResult(t, endpoint3, 1, 5), + pollResult(t, endpoint1, 10, 0), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4), + pollResult(t, endpoint1, 13, 0), pollResult(t, endpoint2, 3, 1), pollResult(t, endpoint3, 0, 4), + pollResult(t, endpoint1, 15, 0), pollResult(t, endpoint2, 1, 3), pollResult(t, endpoint3, 1, 5), } for _, results := range pollResults { @@ -345,7 +363,8 @@ func TestCircuitBreakerManager_Config(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -402,7 +421,8 @@ func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -443,7 +463,8 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -489,7 +510,8 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -547,7 +569,8 @@ func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -604,7 +627,8 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -683,7 +707,8 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -732,7 +757,8 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, @@ -803,7 +829,8 @@ func TestCircuitBreakerManager_Start(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 50, FailureCount: 5, - SuccessThreshold: 2, + SuccessThreshold: 10, + MinimumRequestCount: 10, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go index 34dd355f15..9671573ea9 100644 --- a/pkg/circuit_breaker/config.go +++ b/pkg/circuit_breaker/config.go @@ -61,8 +61,6 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - fmt.Printf("%+v\n", c.MinimumRequestCount) - if c.MinimumRequestCount < 10 { errs.WriteString("MinimumRequestCount must be greater than 10") errs.WriteString("; ") @@ -73,8 +71,8 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - if c.SuccessThreshold == 0 { - errs.WriteString("SuccessThreshold must be greater than 0") + if c.SuccessThreshold == 0 || c.SuccessThreshold > 100 { + errs.WriteString("SuccessThreshold must be between 1 and 100") errs.WriteString("; ") } diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index cad5da5230..ff3bc07692 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -72,10 +72,10 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { BreakerTimeout: 30, FailureThreshold: 5, FailureCount: 5, - SuccessThreshold: 0, + SuccessThreshold: 150, }, wantErr: true, - err: "SuccessThreshold must be greater than 0", + err: "SuccessThreshold must be between 1 and 100", }, { name: "Invalid ObservabilityWindow", diff --git a/pkg/circuit_breaker/store.go b/pkg/circuit_breaker/store.go index ee806e095f..10fb37a8b9 100644 --- a/pkg/circuit_breaker/store.go +++ b/pkg/circuit_breaker/store.go @@ -160,7 +160,7 @@ func (t *TestStore) GetOne(_ context.Context, s string) (string, error) { } vv := res.String() - if vv != "" { + if vv == "" { return "", errors.New("an error occurred decoding the circuit breaker") } diff --git a/pkg/circuit_breaker/store_test.go b/pkg/circuit_breaker/store_test.go index 701aed6209..cee0a9d49f 100644 --- a/pkg/circuit_breaker/store_test.go +++ b/pkg/circuit_breaker/store_test.go @@ -2,8 +2,6 @@ package circuit_breaker import ( "context" - "fmt" - "sync" "testing" "time" @@ -270,29 +268,3 @@ func TestTestStore_SetMany(t *testing.T) { require.Equal(t, cb, storedCB) } } - -func TestTestStore_Concurrency(t *testing.T) { - store := NewTestStore() - ctx := context.Background() - wg := &sync.WaitGroup{} - - // Test concurrent reads - for i := 0; i < 100; i++ { - wg.Add(1) - key := fmt.Sprintf("key_%d", i) - err := store.SetOne(ctx, key, CircuitBreaker{Key: key, State: StateClosed}, time.Minute) - require.NoError(t, err) - } - - go func() { - for i := 0; i < 100; i++ { - _, err := store.GetOne(ctx, fmt.Sprintf("key_%d", i)) - require.NoError(t, err) - wg.Done() - } - }() - - // If there's a race condition, this test might panic or deadlock - time.Sleep(100 * time.Millisecond) - wg.Wait() -} diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index bd94fa3a30..bf81e8f482 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -964,6 +964,7 @@ func TestProcessEventDelivery(t *testing.T) { FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 49269e277c..b1e5c31054 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -1071,6 +1071,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, + MinimumRequestCount: 10, NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } From 96ed4b828bf3c03d724e6766d9f07a63d51d07b2 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 24 Sep 2024 12:49:49 +0200 Subject: [PATCH 30/48] fix: fix lint error --- database/postgres/delivery_attempts.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index bfb37634f7..f33da24e26 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "errors" - "fmt" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/circuit_breaker" @@ -134,7 +133,6 @@ func (d *deliveryAttemptRepo) DeleteProjectDeliveriesAttempts(ctx context.Contex func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lookBackDuration uint64, resetTimes map[string]time.Time) (map[string]circuit_breaker.PollResult, error) { resultsMap := map[string]circuit_breaker.PollResult{} - fmt.Printf("rowValue: %+v\n", resetTimes) query := ` SELECT @@ -173,14 +171,12 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo var rowValue circuit_breaker.PollResult err = d.db.QueryRowxContext(ctx, query2, k, t).StructScan(&rowValue) if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - return nil, err + if errors.Is(err, sql.ErrNoRows) { + continue } } - if &rowValue != nil { - resultsMap[k] = rowValue - } + resultsMap[k] = rowValue } return resultsMap, nil From 5b2c43bf7d40a0565e3c6c582865c1baf882b2ff Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 25 Sep 2024 17:23:52 +0200 Subject: [PATCH 31/48] chore: fix posthog js version --- web/ui/dashboard/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/ui/dashboard/package.json b/web/ui/dashboard/package.json index 3e660e7f25..bc4a14a3e7 100644 --- a/web/ui/dashboard/package.json +++ b/web/ui/dashboard/package.json @@ -28,7 +28,7 @@ "axios": "^1.6.7", "date-fns": "^2.27.0", "monaco-editor": "^0.34.1", - "posthog-js": "^1.36.0", + "posthog-js": "1.36.0", "prismjs": "^1.29.0", "rxjs": "^6.6.0", "tslib": "^2.3.1", From ba678bdffefaffdeb5b526d0ff5ce9d3b63dd356 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 27 Sep 2024 15:55:32 +0200 Subject: [PATCH 32/48] chore: PR review changes --- cmd/agent/agent.go | 16 ++++++++-------- cmd/worker/worker.go | 16 ++++++++-------- database/postgres/configuration.go | 18 +++++++++--------- datastore/models.go | 8 ++++---- sql/1724236900.sql | 4 ++++ 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index 8b53d3f629..a73574d7e9 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -122,27 +122,27 @@ func AddAgentCommand(a *cli.App) *cobra.Command { } func startServerComponent(_ context.Context, a *cli.App) error { + lo := a.Logger.(*log.Logger) + lo.SetPrefix("agent") + cfg, err := config.Get() if err != nil { - a.Logger.WithError(err).Fatal("Failed to load configuration") + lo.WithError(err).Fatal("Failed to load configuration") } start := time.Now() - a.Logger.Info("Starting Convoy data plane ...") + lo.Info("Starting Convoy data plane") apiKeyRepo := postgres.NewAPIKeyRepo(a.DB, a.Cache) userRepo := postgres.NewUserRepo(a.DB, a.Cache) portalLinkRepo := postgres.NewPortalLinkRepo(a.DB, a.Cache) err = realm_chain.Init(&cfg.Auth, apiKeyRepo, userRepo, portalLinkRepo, a.Cache) if err != nil { - a.Logger.WithError(err).Fatal("failed to initialize realm chain") + lo.WithError(err).Fatal("failed to initialize realm chain") } flag := fflag.NewFFlag(&cfg) - lo := a.Logger.(*log.Logger) - lo.SetPrefix("api server") - lvl, err := log.ParseLevel(cfg.Logger.Level) if err != nil { return err @@ -168,7 +168,7 @@ func startServerComponent(_ context.Context, a *cli.App) error { srv.SetHandler(evHandler.BuildDataPlaneRoutes()) - log.Infof("Started convoy server in %s\n", time.Since(start)) + lo.Infof("Started convoy server in %s", time.Since(start)) httpConfig := cfg.Server.HTTP if httpConfig.SSL { @@ -177,7 +177,7 @@ func startServerComponent(_ context.Context, a *cli.App) error { return nil } - log.Println("Starting Convoy Agent on port %v\n", cfg.Server.HTTP.AgentPort) + lo.Infof("Starting Convoy Agent on port %v", cfg.Server.HTTP.AgentPort) go func() { srv.Listen() diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 0ad1b97c27..25a4bcf9e0 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -133,7 +133,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte sc, err := smtp.NewClient(&cfg.SMTP) if err != nil { - a.Logger.WithError(err).Error("Failed to create smtp client") + lo.WithError(err).Error("Failed to create smtp client") return err } @@ -227,11 +227,11 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte configuration, err := configRepo.LoadConfiguration(context.Background()) if err != nil { - a.Logger.WithError(err).Fatal("Failed to instance configuration") + lo.WithError(err).Fatal("Failed to instance configuration") return err } - subscriptionsLoader := loader.NewSubscriptionLoader(subRepo, projectRepo, a.Logger, 0) + subscriptionsLoader := loader.NewSubscriptionLoader(subRepo, projectRepo, lo, 0) subscriptionsTable := memorystore.NewTable(memorystore.OptionSyncer(subscriptionsLoader)) err = memorystore.DefaultStore.Register("subscriptions", subscriptionsTable) @@ -247,14 +247,14 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte go memorystore.DefaultStore.Sync(ctx, interval) - newTelemetry := telemetry.NewTelemetry(a.Logger.(*log.Logger), configuration, + newTelemetry := telemetry.NewTelemetry(lo, configuration, telemetry.OptionTracker(counter), telemetry.OptionBackend(pb), telemetry.OptionBackend(mb)) dispatcher, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, a.Licenser, false) if err != nil { - a.Logger.WithError(err).Fatal("Failed to create new net dispatcher") + lo.WithError(err).Fatal("Failed to create new net dispatcher") return err } @@ -272,15 +272,15 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte }), ) if err != nil { - a.Logger.WithError(err).Fatal("Failed to create circuit breaker manager") + lo.WithError(err).Fatal("Failed to create circuit breaker manager") } go circuitBreakerManager.Start(ctx, attemptRepo.GetFailureAndSuccessCounts) } else { - a.Logger.Warn(fflag.ErrCircuitBreakerNotEnabled) + lo.Warn(fflag.ErrCircuitBreakerNotEnabled) } - channels := make(map[string]task.EventChannel) + channels := make(map[string]task.EventChannel) defaultCh, broadcastCh, dynamicCh := task.NewDefaultEventChannel(), task.NewBroadcastEventChannel(subscriptionsTable), task.NewDynamicEventChannel() channels["default"] = defaultCh channels["broadcast"] = broadcastCh diff --git a/database/postgres/configuration.go b/database/postgres/configuration.go index 845163ff21..84ee70b37c 100644 --- a/database/postgres/configuration.go +++ b/database/postgres/configuration.go @@ -45,15 +45,15 @@ const ( s3_session_token AS "storage_policy.s3.session_token", s3_endpoint AS "storage_policy.s3.endpoint", s3_prefix AS "storage_policy.s3.prefix", - cb_sample_rate AS "breaker_config.sample_rate", - cb_error_timeout AS "breaker_config.error_timeout", - cb_failure_threshold AS "breaker_config.failure_threshold", - cb_failure_count AS "breaker_config.failure_count", - cb_success_threshold AS "breaker_config.success_threshold", - cb_observability_window AS "breaker_config.observability_window", - cb_minimum_request_count as "breaker_config.minimum_request_count", - cb_notification_thresholds::INTEGER[] AS "breaker_config.notification_thresholds", - cb_consecutive_failure_threshold AS "breaker_config.consecutive_failure_threshold", + cb_sample_rate AS "circuit_breaker.sample_rate", + cb_error_timeout AS "circuit_breaker.error_timeout", + cb_failure_threshold AS "circuit_breaker.failure_threshold", + cb_failure_count AS "circuit_breaker.failure_count", + cb_success_threshold AS "circuit_breaker.success_threshold", + cb_observability_window AS "circuit_breaker.observability_window", + cb_minimum_request_count as "circuit_breaker.minimum_request_count", + cb_notification_thresholds::INTEGER[] AS "circuit_breaker.notification_thresholds", + cb_consecutive_failure_threshold AS "circuit_breaker.consecutive_failure_threshold", created_at, updated_at, deleted_at diff --git a/datastore/models.go b/datastore/models.go index 72ea66e482..010b81cff0 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" "fmt" - "github.com/frain-dev/convoy/pkg/circuit_breaker" + cb "github.com/frain-dev/convoy/pkg/circuit_breaker" "math" "net/http" "strings" @@ -1342,7 +1342,7 @@ type Configuration struct { StoragePolicy *StoragePolicyConfiguration `json:"storage_policy" db:"storage_policy"` RetentionPolicy *RetentionPolicyConfiguration `json:"retention_policy" db:"retention_policy"` - CircuitBreakerConfig *CircuitBreakerConfig `json:"breaker_config" db:"breaker_config"` + CircuitBreakerConfig *CircuitBreakerConfig `json:"circuit_breaker" db:"circuit_breaker"` CreatedAt time.Time `json:"created_at,omitempty" db:"created_at,omitempty" swaggertype:"string"` UpdatedAt time.Time `json:"updated_at,omitempty" db:"updated_at,omitempty" swaggertype:"string"` @@ -1356,13 +1356,13 @@ func (c *Configuration) GetCircuitBreakerConfig() CircuitBreakerConfig { return CircuitBreakerConfig{} } -func (c *Configuration) ToCircuitBreakerConfig() *circuit_breaker.CircuitBreakerConfig { +func (c *Configuration) ToCircuitBreakerConfig() *cb.CircuitBreakerConfig { notificationThresholds := [3]uint64{} for i := range c.CircuitBreakerConfig.NotificationThresholds { notificationThresholds[i] = uint64(c.CircuitBreakerConfig.NotificationThresholds[i]) } - return &circuit_breaker.CircuitBreakerConfig{ + return &cb.CircuitBreakerConfig{ SampleRate: c.CircuitBreakerConfig.SampleRate, FailureCount: c.CircuitBreakerConfig.FailureCount, BreakerTimeout: c.CircuitBreakerConfig.ErrorTimeout, diff --git a/sql/1724236900.sql b/sql/1724236900.sql index 8f54fb2619..134e5b5f7d 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -21,3 +21,7 @@ alter table convoy.configurations drop column if exists cb_success_threshold; alter table convoy.configurations drop column if exists cb_observability_window; alter table convoy.configurations drop column if exists cb_notification_thresholds; alter table convoy.configurations drop column if exists cb_consecutive_failure_threshold; +drop index if exists convoy.idx_delivery_attempts_created_at; +drop index if exists convoy.idx_delivery_attempts_event_delivery_id_created_at; +drop index if exists convoy.idx_delivery_attempts_event_delivery_id; + From e0ecd0770036d10aec1cfb80ab9e41e9b3058e38 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 27 Sep 2024 19:43:03 +0200 Subject: [PATCH 33/48] feat: add circuit breaker metrics --- cmd/server/server.go | 2 +- cmd/worker/worker.go | 2 +- internal/pkg/metrics/metrics.go | 9 +- .../circuit_breaker_collector.go | 158 ++++++++++++++++++ pkg/circuit_breaker/metrics.go | 31 ---- 5 files changed, 167 insertions(+), 35 deletions(-) create mode 100644 pkg/circuit_breaker/circuit_breaker_collector.go delete mode 100644 pkg/circuit_breaker/metrics.go diff --git a/cmd/server/server.go b/cmd/server/server.go index af2992513f..bca0336118 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -154,7 +154,7 @@ func startConvoyServer(a *cli.App) error { s.RegisterTask("0 0 * * *", convoy.ScheduleQueue, convoy.RetentionPolicies) s.RegisterTask("0 * * * *", convoy.ScheduleQueue, convoy.TokenizeSearch) - metrics.RegisterQueueMetrics(a.Queue, a.DB) + metrics.RegisterQueueMetrics(a.Queue, a.DB, nil) // Start scheduler s.Start() diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 25a4bcf9e0..86fc5806e1 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -378,7 +378,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher), nil) consumer.RegisterHandlers(convoy.DeleteArchivedTasksProcessor, task.DeleteArchivedTasks(a.Queue, rd), nil) - metrics.RegisterQueueMetrics(a.Queue, a.DB) + metrics.RegisterQueueMetrics(a.Queue, a.DB, circuitBreakerManager) // start worker consumer.Start() diff --git a/internal/pkg/metrics/metrics.go b/internal/pkg/metrics/metrics.go index a6f1ad6965..476c7b8d96 100644 --- a/internal/pkg/metrics/metrics.go +++ b/internal/pkg/metrics/metrics.go @@ -4,6 +4,7 @@ import ( "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/database/postgres" + cb "github.com/frain-dev/convoy/pkg/circuit_breaker" "sync" "github.com/frain-dev/convoy/queue" @@ -29,9 +30,13 @@ func Reset() { prometheus.DefaultRegisterer = prometheus.NewRegistry() } -func RegisterQueueMetrics(q queue.Queuer, db database.Database) { +func RegisterQueueMetrics(q queue.Queuer, db database.Database, cbm *cb.CircuitBreakerManager) { configuration, err := config.Get() if err == nil && configuration.Metrics.IsEnabled { - Reg().MustRegister(q.(*redisqueue.RedisQueue), db.(*postgres.Postgres)) + if cbm == nil { // cbm can be nil if the feature flag is not enabled + Reg().MustRegister(q.(*redisqueue.RedisQueue), db.(*postgres.Postgres)) + } else { + Reg().MustRegister(q.(*redisqueue.RedisQueue), db.(*postgres.Postgres), cbm) + } } } diff --git a/pkg/circuit_breaker/circuit_breaker_collector.go b/pkg/circuit_breaker/circuit_breaker_collector.go new file mode 100644 index 0000000000..d983d96cac --- /dev/null +++ b/pkg/circuit_breaker/circuit_breaker_collector.go @@ -0,0 +1,158 @@ +package circuit_breaker + +import ( + "context" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/pkg/log" + "github.com/prometheus/client_golang/prometheus" + "time" +) + +const namespace = "circuit_breaker" + +var ( + cachedMetrics *Metrics + metricsConfig *config.MetricsConfiguration + lastRun = time.Now() + + circuitBreakerState = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "state"), + "The current state of the circuit breaker (0: Closed, 1: Half-Open, 2: Open)", + []string{"key"}, nil, + ) + circuitBreakerRequests = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "requests_total"), + "Total number of requests processed by the circuit breaker", + []string{"key"}, nil, + ) + circuitBreakerFailures = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "failures_total"), + "Total number of failed requests processed by the circuit breaker", + []string{"key"}, nil, + ) + circuitBreakerSuccesses = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "successes_total"), + "Total number of successful requests processed by the circuit breaker", + []string{"key"}, nil, + ) + circuitBreakerFailureRate = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "failure_rate"), + "Current failure rate of the circuit breaker", + []string{"key"}, nil, + ) + circuitBreakerSuccessRate = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "success_rate"), + "Current success rate of the circuit breaker", + []string{"key"}, nil, + ) + circuitBreakerConsecutiveFailures = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "consecutive_failures"), + "Number of consecutive failures for the circuit breaker", + []string{"key"}, nil, + ) + circuitBreakerNotificationsSent = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "notifications_sent"), + "Number of notifications sent by the circuit breaker", + []string{"key"}, nil, + ) +) + +type Metrics struct { + circuitBreakers []CircuitBreaker +} + +func (cb *CircuitBreakerManager) collectMetrics() (*Metrics, error) { + metrics := &Metrics{} + cbs, err := cb.loadCircuitBreakers(context.Background()) + if err != nil { + return metrics, err + } + + metrics.circuitBreakers = cbs + + return metrics, nil +} + +func (cb *CircuitBreakerManager) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(cb, ch) +} + +func (cb *CircuitBreakerManager) Collect(ch chan<- prometheus.Metric) { + if metricsConfig == nil { + cfg, err := config.Get() + if err != nil { + return + } + metricsConfig = &cfg.Metrics + } + if !metricsConfig.IsEnabled { + return + } + + var metrics *Metrics + var err error + now := time.Now() + if cachedMetrics != nil && lastRun.Add(time.Duration(metricsConfig.Prometheus.SampleTime)*time.Second).After(now) { + metrics = cachedMetrics + } else { + metrics, err = cb.collectMetrics() + if err != nil { + log.Errorf("Failed to collect metrics data: %v", err) + return + } + cachedMetrics = metrics + } + + for _, metric := range metrics.circuitBreakers { + ch <- prometheus.MustNewConstMetric( + circuitBreakerState, + prometheus.GaugeValue, + float64(metric.State), + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerRequests, + prometheus.CounterValue, + float64(metric.Requests), + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerFailures, + prometheus.CounterValue, + float64(metric.TotalFailures), + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerSuccesses, + prometheus.CounterValue, + float64(metric.TotalSuccesses), + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerFailureRate, + prometheus.GaugeValue, + metric.FailureRate, + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerSuccessRate, + prometheus.GaugeValue, + metric.SuccessRate, + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerConsecutiveFailures, + prometheus.GaugeValue, + float64(metric.ConsecutiveFailures), + metric.Key, + ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerNotificationsSent, + prometheus.GaugeValue, + float64(metric.NotificationsSent), + metric.Key, + ) + } + + lastRun = now +} diff --git a/pkg/circuit_breaker/metrics.go b/pkg/circuit_breaker/metrics.go deleted file mode 100644 index 2d8270515e..0000000000 --- a/pkg/circuit_breaker/metrics.go +++ /dev/null @@ -1,31 +0,0 @@ -package circuit_breaker - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - circuitBreakerState = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "circuit_breaker_state", - Help: "The current state of the circuit breaker (0: Closed, 1: Half-Open, 2: Open)", - }, - []string{"key"}, - ) - - circuitBreakerRequests = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "circuit_breaker_requests_total", - Help: "The total number of requests processed by the circuit breaker", - }, - []string{"key", "result"}, - ) -) - -func (cb *CircuitBreakerManager) UpdateMetrics(breaker CircuitBreaker) { - // todo(raymond) call UpdateMetrics in the sampleStore method after updating each circuit breaker - circuitBreakerState.WithLabelValues(breaker.Key).Set(float64(breaker.State)) - circuitBreakerRequests.WithLabelValues(breaker.Key, "success").Add(float64(breaker.TotalSuccesses)) - circuitBreakerRequests.WithLabelValues(breaker.Key, "failure").Add(float64(breaker.TotalFailures)) -} From 9c7490b3c8359dbbcb84402258dfb78982a81a3e Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 30 Sep 2024 14:37:46 +0200 Subject: [PATCH 34/48] feat: add tenant id --- database/postgres/delivery_attempts.go | 6 +++-- pkg/circuit_breaker/circuit_breaker.go | 5 +++- .../circuit_breaker_collector.go | 24 ++++++++++++------- .../circuit_breaker_manager.go | 9 ++++--- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index f33da24e26..01538b5110 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -137,10 +137,11 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo query := ` SELECT endpoint_id AS key, + project_id AS tenant_id, COUNT(CASE WHEN status = false THEN 1 END) AS failures, COUNT(CASE WHEN status = true THEN 1 END) AS successes FROM convoy.delivery_attempts - WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) group by endpoint_id; + WHERE created_at >= NOW() - MAKE_INTERVAL(hours := 1000) group by endpoint_id, project_id; ` rows, err := d.db.QueryxContext(ctx, query, lookBackDuration) @@ -161,10 +162,11 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo query2 := ` SELECT endpoint_id AS key, + project_id AS tenant_id, COUNT(CASE WHEN status = false THEN 1 END) AS failures, COUNT(CASE WHEN status = true THEN 1 END) AS successes FROM convoy.delivery_attempts - WHERE endpoint_id = $1 AND created_at >= $2 group by endpoint_id; + WHERE endpoint_id = $1 AND created_at >= $2 group by endpoint_id, project_id; ` for k, t := range resetTimes { diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 1a48da233d..2a2a01a99f 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -10,6 +10,8 @@ import ( type CircuitBreaker struct { // Circuit breaker key Key string `json:"key"` + // Circuit breaker tenant id + TenantId string `json:"tenant_id"` // Circuit breaker state State State `json:"state"` // Number of requests in the observability window @@ -30,9 +32,10 @@ type CircuitBreaker struct { NotificationsSent uint64 `json:"notifications_sent"` } -func NewCircuitBreaker(key string) *CircuitBreaker { +func NewCircuitBreaker(key string, tenantId string) *CircuitBreaker { return &CircuitBreaker{ Key: key, + TenantId: tenantId, State: StateClosed, NotificationsSent: 0, } diff --git a/pkg/circuit_breaker/circuit_breaker_collector.go b/pkg/circuit_breaker/circuit_breaker_collector.go index d983d96cac..f76888e875 100644 --- a/pkg/circuit_breaker/circuit_breaker_collector.go +++ b/pkg/circuit_breaker/circuit_breaker_collector.go @@ -18,42 +18,42 @@ var ( circuitBreakerState = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "state"), "The current state of the circuit breaker (0: Closed, 1: Half-Open, 2: Open)", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerRequests = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "requests_total"), "Total number of requests processed by the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerFailures = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "failures_total"), "Total number of failed requests processed by the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerSuccesses = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "successes_total"), "Total number of successful requests processed by the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerFailureRate = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "failure_rate"), "Current failure rate of the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerSuccessRate = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "success_rate"), "Current success rate of the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerConsecutiveFailures = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "consecutive_failures"), "Number of consecutive failures for the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) circuitBreakerNotificationsSent = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "notifications_sent"), "Number of notifications sent by the circuit breaker", - []string{"key"}, nil, + []string{"key", "tenant_id"}, nil, ) ) @@ -109,48 +109,56 @@ func (cb *CircuitBreakerManager) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(metric.State), metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerRequests, prometheus.CounterValue, float64(metric.Requests), metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerFailures, prometheus.CounterValue, float64(metric.TotalFailures), metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerSuccesses, prometheus.CounterValue, float64(metric.TotalSuccesses), metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerFailureRate, prometheus.GaugeValue, metric.FailureRate, metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerSuccessRate, prometheus.GaugeValue, metric.SuccessRate, metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerConsecutiveFailures, prometheus.GaugeValue, float64(metric.ConsecutiveFailures), metric.Key, + metric.TenantId, ) ch <- prometheus.MustNewConstMetric( circuitBreakerNotificationsSent, prometheus.GaugeValue, float64(metric.NotificationsSent), metric.Key, + metric.TenantId, ) } diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 3140f792aa..b7cfd0b7e4 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -70,6 +70,7 @@ func (s State) String() string { type PollResult struct { Key string `json:"key" db:"key"` + TenantId string `json:"tenant_id" db:"tenant_id"` Failures uint64 `json:"failures" db:"failures"` Successes uint64 `json:"successes" db:"successes"` } @@ -160,8 +161,10 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma circuitBreakers := make(map[string]CircuitBreaker, len(pollResults)) - keys, j := make([]string, len(pollResults)), 0 + keys, tenants, j := make([]string, len(pollResults)), make([]string, len(pollResults)), 0 for k := range pollResults { + tenants[j] = pollResults[k].TenantId + key := fmt.Sprintf("%s%s", prefix, k) keys[j] = key j++ @@ -174,7 +177,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma for i := range res { if res[i] == nil { - circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i]) + circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i]) continue } @@ -183,7 +186,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) // the circuit breaker is corrupted, create a new one in its place - circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i]) + circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i]) continue } From f5393173ea8d77995d1b750f90658f614e27d329 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 30 Sep 2024 14:39:54 +0200 Subject: [PATCH 35/48] chore: add interval variable back --- database/postgres/delivery_attempts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index 01538b5110..6e8bea8800 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -141,7 +141,7 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo COUNT(CASE WHEN status = false THEN 1 END) AS failures, COUNT(CASE WHEN status = true THEN 1 END) AS successes FROM convoy.delivery_attempts - WHERE created_at >= NOW() - MAKE_INTERVAL(hours := 1000) group by endpoint_id, project_id; + WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) group by endpoint_id, project_id; ` rows, err := d.db.QueryxContext(ctx, query, lookBackDuration) From 11de67023af2b11b90fc74a4468efd8fd1f2496b Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 1 Oct 2024 15:00:22 +0200 Subject: [PATCH 36/48] feat: Handle disabling endpoint correctly; Add debug logs for circuit state transition; Remove failure_count config --- cmd/hooks/hooks.go | 1 - cmd/worker/worker.go | 53 ++++++++++++++++++- config/config.go | 2 - config/config_test.go | 3 -- database/postgres/configuration.go | 23 ++++---- datastore/models.go | 3 -- pkg/circuit_breaker/circuit_breaker.go | 6 +++ .../circuit_breaker_manager.go | 32 +++++++---- .../circuit_breaker_manager_test.go | 36 ++++--------- pkg/circuit_breaker/config.go | 9 ---- pkg/circuit_breaker/config_test.go | 19 ------- sql/1724236900.sql | 2 - worker/task/process_event_delivery.go | 17 ------ 13 files changed, 97 insertions(+), 109 deletions(-) diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index 2573e22bf2..c9bc5fe82b 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -334,7 +334,6 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat circuitBreakerConfig := &datastore.CircuitBreakerConfig{ SampleRate: cfg.CircuitBreaker.SampleRate, ErrorTimeout: cfg.CircuitBreaker.ErrorTimeout, - FailureCount: cfg.CircuitBreaker.FailureCount, FailureThreshold: cfg.CircuitBreaker.FailureThreshold, SuccessThreshold: cfg.CircuitBreaker.SuccessThreshold, ObservabilityWindow: cfg.CircuitBreaker.ObservabilityWindow, diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 86fc5806e1..c972d5862a 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -3,8 +3,11 @@ package worker import ( "context" "fmt" + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/notifications" "github.com/frain-dev/convoy/internal/pkg/fflag" "net/http" + "strings" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/config" @@ -266,8 +269,54 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte cb.ConfigOption(configuration.ToCircuitBreakerConfig()), cb.StoreOption(cb.NewRedisStore(rd.Client(), clock.NewRealClock())), cb.ClockOption(clock.NewRealClock()), - cb.NotificationFunctionOption(func(c cb.CircuitBreaker) error { - fmt.Printf("notification: %+v\n", c) + cb.NotificationFunctionOption(func(n cb.NotificationType, c cb.CircuitBreakerConfig, b cb.CircuitBreaker) error { + endpointId := strings.Split(b.Key, ":")[1] + project, funcErr := projectRepo.FetchProjectByID(ctx, b.TenantId) + if funcErr != nil { + return funcErr + } + + endpoint, funcErr := endpointRepo.FindEndpointByID(ctx, endpointId, b.TenantId) + if funcErr != nil { + return funcErr + } + + switch n { + case cb.TypeTriggerThreshold: + innerErr := notifications.SendEndpointNotification(ctx, + endpoint, project, + endpoint.Status, + q, true, + fmt.Sprintf("%+d percent of the events send to endpoint (%s) in project (%s) have failed", + c.NotificationThresholds[b.NotificationsSent], endpoint.Name, project.Name), + fmt.Sprintf("circuit breaker state for %s is %v", endpoint.Name, b.State), + 429) + if innerErr != nil { + return innerErr + } + + break + case cb.TypeDisableResource: + breakerErr := endpointRepo.UpdateEndpointStatus(ctx, b.TenantId, endpointId, datastore.InactiveEndpointStatus) + if breakerErr != nil { + return breakerErr + } + + innerErr := notifications.SendEndpointNotification(ctx, + endpoint, project, + endpoint.Status, + q, true, + fmt.Sprintf("Endpoint (%s)'s circuit breaker in the project (%s) has tripped %d times; the endpoint has been de-activated", + endpoint.Name, project.Name, c.ConsecutiveFailureThreshold), + fmt.Sprintf("circuit breaker state for endpoint (%s) is %v", endpoint.Name, b.State), + 429) + if innerErr != nil { + return innerErr + } + break + default: + return fmt.Errorf("unsupported circuit breaker notification type: %s", n) + } return nil }), ) diff --git a/config/config.go b/config/config.go index 6ace2c82b9..9bd66b4f68 100644 --- a/config/config.go +++ b/config/config.go @@ -78,7 +78,6 @@ var DefaultConfiguration = Configuration{ SampleRate: 30, ErrorTimeout: 30, FailureThreshold: 70, - FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, @@ -274,7 +273,6 @@ type RetentionPolicyConfiguration struct { type CircuitBreakerConfiguration struct { SampleRate uint64 `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` - FailureCount uint64 `json:"failure_count" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_COUNT"` ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` FailureThreshold uint64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` diff --git a/config/config_test.go b/config/config_test.go index f2cbb14b35..e99e01b77b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -129,7 +129,6 @@ func TestLoadConfig(t *testing.T) { SampleRate: 30, ErrorTimeout: 30, FailureThreshold: 70, - FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, @@ -211,7 +210,6 @@ func TestLoadConfig(t *testing.T) { SampleRate: 30, ErrorTimeout: 30, FailureThreshold: 70, - FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, @@ -288,7 +286,6 @@ func TestLoadConfig(t *testing.T) { SampleRate: 30, ErrorTimeout: 30, FailureThreshold: 70, - FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, diff --git a/database/postgres/configuration.go b/database/postgres/configuration.go index 84ee70b37c..d945a3796b 100644 --- a/database/postgres/configuration.go +++ b/database/postgres/configuration.go @@ -21,12 +21,11 @@ const ( s3_region, s3_session_token, s3_endpoint, retention_policy_policy, retention_policy_enabled, cb_sample_rate,cb_error_timeout, - cb_failure_threshold,cb_failure_count, - cb_success_threshold,cb_observability_window, - cb_notification_thresholds,cb_consecutive_failure_threshold, - cb_minimum_request_count + cb_failure_threshold, cb_success_threshold, + cb_observability_window, cb_notification_thresholds, + cb_consecutive_failure_threshold, cb_minimum_request_count ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23); + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22); ` fetchConfiguration = ` @@ -48,7 +47,6 @@ const ( cb_sample_rate AS "circuit_breaker.sample_rate", cb_error_timeout AS "circuit_breaker.error_timeout", cb_failure_threshold AS "circuit_breaker.failure_threshold", - cb_failure_count AS "circuit_breaker.failure_count", cb_success_threshold AS "circuit_breaker.success_threshold", cb_observability_window AS "circuit_breaker.observability_window", cb_minimum_request_count as "circuit_breaker.minimum_request_count", @@ -81,12 +79,11 @@ const ( cb_sample_rate = $15, cb_error_timeout = $16, cb_failure_threshold = $17, - cb_failure_count = $18, - cb_success_threshold = $19, - cb_observability_window = $20, - cb_notification_thresholds = $21, - cb_consecutive_failure_threshold = $22, - cb_minimum_request_count = $23, + cb_success_threshold = $18, + cb_observability_window = $19, + cb_notification_thresholds = $20, + cb_consecutive_failure_threshold = $21, + cb_minimum_request_count = $22, updated_at = NOW() WHERE id = $1 AND deleted_at IS NULL; ` @@ -138,7 +135,6 @@ func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore. cb.SampleRate, cb.ErrorTimeout, cb.FailureThreshold, - cb.FailureCount, cb.SuccessThreshold, cb.ObservabilityWindow, cb.NotificationThresholds, @@ -212,7 +208,6 @@ func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Con cb.SampleRate, cb.ErrorTimeout, cb.FailureThreshold, - cb.FailureCount, cb.SuccessThreshold, cb.ObservabilityWindow, cb.NotificationThresholds, diff --git a/datastore/models.go b/datastore/models.go index 010b81cff0..168b0bc86b 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -323,7 +323,6 @@ var ( SampleRate: 30, ErrorTimeout: 30, FailureThreshold: 70, - FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, NotificationThresholds: pq.Int64Array{10, 30, 50}, @@ -1364,7 +1363,6 @@ func (c *Configuration) ToCircuitBreakerConfig() *cb.CircuitBreakerConfig { return &cb.CircuitBreakerConfig{ SampleRate: c.CircuitBreakerConfig.SampleRate, - FailureCount: c.CircuitBreakerConfig.FailureCount, BreakerTimeout: c.CircuitBreakerConfig.ErrorTimeout, FailureThreshold: c.CircuitBreakerConfig.FailureThreshold, SuccessThreshold: c.CircuitBreakerConfig.SuccessThreshold, @@ -1405,7 +1403,6 @@ type OnPremStorage struct { type CircuitBreakerConfig struct { SampleRate uint64 `json:"sample_rate" db:"sample_rate"` ErrorTimeout uint64 `json:"error_timeout" db:"error_timeout"` - FailureCount uint64 `json:"failure_count" db:"failure_count"` FailureThreshold uint64 `json:"failure_threshold" db:"failure_threshold"` SuccessThreshold uint64 `json:"success_threshold" db:"success_threshold"` ObservabilityWindow uint64 `json:"observability_window" db:"observability_window"` diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 2a2a01a99f..426f45e955 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -55,14 +55,20 @@ func (b *CircuitBreaker) tripCircuitBreaker(resetTime time.Time) { b.State = StateOpen b.WillResetAt = resetTime b.ConsecutiveFailures++ + log.Infof("[circuit breaker] circuit breaker transitioned from closed to open.") + log.Debugf("[circuit breaker] circuit breaker state: %+v", b) } func (b *CircuitBreaker) toHalfOpen() { b.State = StateHalfOpen + log.Infof("[circuit breaker] circuit breaker transitioned from open to half-open") + log.Debugf("[circuit breaker] circuit breaker state: %+v", b) } func (b *CircuitBreaker) resetCircuitBreaker() { b.State = StateClosed b.NotificationsSent = 0 b.ConsecutiveFailures = 0 + log.Infof("[circuit breaker] circuit breaker transitioned from half-open to closed") + log.Debugf("[circuit breaker] circuit breaker state: %+v", b) } diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index b7cfd0b7e4..65b3133080 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -11,11 +11,6 @@ import ( "time" ) -// todo(raymond): send notifications when notification thresholds are hit, then update breaker state -// todo(raymond): save the previous failure rate along side the current so we can compare to see if it's reducing -// todo(raymond): metrics should contain error rate -// todo(raymond): use a guage for failure rate metrics - const prefix = "breaker:" const mutexKey = "convoy:circuit_breaker:mutex" @@ -47,6 +42,7 @@ var ( // State represents a state of a CircuitBreaker. type State int +type NotificationType string // These are the states of a CircuitBreaker. const ( @@ -55,6 +51,11 @@ const ( StateOpen ) +const ( + TypeDisableResource NotificationType = "disable" + TypeTriggerThreshold NotificationType = "trigger" +) + func (s State) String() string { switch s { case StateClosed: @@ -79,7 +80,7 @@ type CircuitBreakerManager struct { config *CircuitBreakerConfig clock clock.Clock store CircuitBreakerStore - notificationFn func(CircuitBreaker) error + notificationFn func(NotificationType, CircuitBreakerConfig, CircuitBreaker) error } func NewCircuitBreakerManager(options ...CircuitBreakerOption) (*CircuitBreakerManager, error) { @@ -144,7 +145,7 @@ func ConfigOption(config *CircuitBreakerConfig) CircuitBreakerOption { } } -func NotificationFunctionOption(fn func(c CircuitBreaker) error) CircuitBreakerOption { +func NotificationFunctionOption(fn func(NotificationType, CircuitBreakerConfig, CircuitBreaker) error) CircuitBreakerOption { return func(cb *CircuitBreakerManager) error { if fn == nil { return ErrNotificationFunctionMustNotBeNil @@ -220,7 +221,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma if breaker.State == StateHalfOpen && breaker.SuccessRate >= float64(cb.config.SuccessThreshold) { breaker.resetCircuitBreaker() } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && breaker.Requests >= cb.config.MinimumRequestCount { - if breaker.FailureRate >= float64(cb.config.FailureThreshold) || breaker.TotalFailures >= cb.config.FailureCount { + if breaker.FailureRate >= float64(cb.config.FailureThreshold) { breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) } } @@ -233,13 +234,22 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma if cb.notificationFn != nil { if prevFailureRate < breaker.FailureRate && breaker.NotificationsSent < 3 { if breaker.FailureRate >= float64(cb.config.NotificationThresholds[breaker.NotificationsSent]) { - innerErr := cb.notificationFn(breaker) + innerErr := cb.notificationFn(TypeTriggerThreshold, *cb.config, breaker) if innerErr != nil { - log.WithError(innerErr).Errorf("[circuit breaker] failed to execute notification function") + log.WithError(innerErr).Errorf("[circuit breaker] failed to execute threshold notification function") } + log.Debugf("[circuit breaker] executed threshold notification function at %v", cb.config.NotificationThresholds[breaker.NotificationsSent]) breaker.NotificationsSent++ } } + + if breaker.ConsecutiveFailures > cb.GetConfig().ConsecutiveFailureThreshold { + innerErr := cb.notificationFn(TypeDisableResource, *cb.config, breaker) + if innerErr != nil { + log.WithError(innerErr).Errorf("[circuit breaker] failed to execute disable resource notification function") + } + log.Debug("[circuit breaker] executed disable resource notification function") + } } circuitBreakers[key] = breaker @@ -305,7 +315,7 @@ func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error case StateOpen: return ErrOpenState case StateHalfOpen: - if b.TotalFailures > cb.config.FailureCount { + if b.FailureRate > float64(cb.config.FailureThreshold) { return ErrTooManyRequests } return nil diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index 44c920ed6d..90e16fbbda 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -62,7 +62,6 @@ func TestCircuitBreakerManager(t *testing.T) { SampleRate: 2, BreakerTimeout: 30, FailureThreshold: 70, - FailureCount: 3, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -75,11 +74,11 @@ func TestCircuitBreakerManager(t *testing.T) { ClockOption(testClock), StoreOption(store), ConfigOption(c), - NotificationFunctionOption(func(c CircuitBreaker) error { + NotificationFunctionOption(func(n NotificationType, c CircuitBreakerConfig, b CircuitBreaker) error { triggered = append(triggered, notificationTriggered{ - State: c.State, - Rate: c.FailureRate, - Sent: c.NotificationsSent, + State: b.State, + Rate: b.FailureRate, + Sent: b.NotificationsSent, }) return nil }), @@ -136,7 +135,6 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { SampleRate: 2, BreakerTimeout: 30, FailureThreshold: 70, - FailureCount: 3, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -192,7 +190,6 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { SampleRate: 2, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 3, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -205,7 +202,6 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { endpointId := "endpoint-1" pollResults := []map[string]PollResult{ pollResult(t, endpointId, 1, 2), // Closed - pollResult(t, endpointId, 13, 1), // Open (FailureCount reached) pollResult(t, endpointId, 13, 1), // Still Open pollResult(t, endpointId, 10, 1), // Half-Open (after ErrorTimeout) pollResult(t, endpointId, 0, 2), // Closed (SuccessThreshold reached) @@ -215,7 +211,6 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { expectedStates := []State{ StateClosed, StateOpen, - StateOpen, StateHalfOpen, StateClosed, StateOpen, @@ -230,7 +225,7 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { require.Equal(t, expectedStates[i], breaker.State, "Iteration %d: expected state %v, got %v", i, expectedStates[i], breaker.State) - if i == 2 { + if i == 1 { // Advance time to trigger the transition to half-open testClock.AdvanceTime(time.Duration(c.BreakerTimeout+1) * time.Second) } else { @@ -261,7 +256,6 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { SampleRate: 2, BreakerTimeout: 30, FailureThreshold: 70, - FailureCount: 3, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -315,7 +309,6 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { SampleRate: 2, BreakerTimeout: 30, FailureThreshold: 60, - FailureCount: 3, SuccessThreshold: 10, ObservabilityWindow: 5, MinimumRequestCount: 10, @@ -362,7 +355,6 @@ func TestCircuitBreakerManager_Config(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -420,7 +412,6 @@ func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -437,13 +428,13 @@ func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { }) t.Run("Half-Open State with Too Many Failures", func(t *testing.T) { - breaker := CircuitBreaker{State: StateHalfOpen, TotalFailures: 6} + breaker := CircuitBreaker{State: StateHalfOpen, FailureRate: 60} err := manager.getCircuitBreakerError(breaker) require.Equal(t, ErrTooManyRequests, err) }) t.Run("Half-Open State with Acceptable Failures", func(t *testing.T) { - breaker := CircuitBreaker{State: StateHalfOpen, TotalFailures: 4} + breaker := CircuitBreaker{State: StateHalfOpen, FailureRate: 40} err := manager.getCircuitBreakerError(breaker) require.NoError(t, err) }) @@ -462,7 +453,6 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -509,7 +499,6 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -568,7 +557,6 @@ func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -626,7 +614,6 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -674,9 +661,9 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { t.Run("Half-Open State with Too Many Failures", func(t *testing.T) { cb := CircuitBreaker{ - Key: "test_half_open", - State: StateHalfOpen, - TotalFailures: 6, + Key: "test_half_open", + State: StateHalfOpen, + FailureRate: 60, } err := manager.store.SetOne(ctx, "breaker:test_half_open", cb, time.Minute) require.NoError(t, err) @@ -706,7 +693,6 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -756,7 +742,6 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, @@ -828,7 +813,6 @@ func TestCircuitBreakerManager_Start(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go index 9671573ea9..5ea836c45c 100644 --- a/pkg/circuit_breaker/config.go +++ b/pkg/circuit_breaker/config.go @@ -23,10 +23,6 @@ type CircuitBreakerConfig struct { // that will trip a circuit breaker MinimumRequestCount uint64 `json:"request_count"` - // FailureCount total number of failed requests in the observability window - // that will trip a circuit breaker - FailureCount uint64 `json:"failure_count"` - // SuccessThreshold is the % of successful requests in the observability window // after which a circuit breaker in the half-open state will go into the closed state SuccessThreshold uint64 `json:"success_threshold"` @@ -66,11 +62,6 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - if c.FailureCount == 0 { - errs.WriteString("FailureCount must be greater than 0") - errs.WriteString("; ") - } - if c.SuccessThreshold == 0 || c.SuccessThreshold > 100 { errs.WriteString("SuccessThreshold must be between 1 and 100") errs.WriteString("; ") diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index ff3bc07692..dcdff4113e 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -18,7 +18,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, @@ -54,24 +53,12 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { wantErr: true, err: "FailureThreshold must be between 1 and 100", }, - { - name: "Invalid FailureCount", - config: CircuitBreakerConfig{ - SampleRate: 1, - BreakerTimeout: 30, - FailureThreshold: 50, - FailureCount: 0, - }, - wantErr: true, - err: "FailureCount must be greater than 0", - }, { name: "Invalid SuccessThreshold", config: CircuitBreakerConfig{ SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 5, - FailureCount: 5, SuccessThreshold: 150, }, wantErr: true, @@ -83,7 +70,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 0, NotificationThresholds: [3]uint64{10, 20, 30}, @@ -97,7 +83,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 200, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 1, NotificationThresholds: [3]uint64{10, 20, 30}, @@ -111,7 +96,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 5, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{}, @@ -125,7 +109,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{10, 20, 30}, @@ -140,7 +123,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 30, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, NotificationThresholds: [3]uint64{30, 50, 60}, @@ -153,7 +135,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { name: "Invalid MinimumRequestCount", config: CircuitBreakerConfig{ SampleRate: 1, - FailureCount: 5, BreakerTimeout: 30, FailureThreshold: 30, SuccessThreshold: 2, diff --git a/sql/1724236900.sql b/sql/1724236900.sql index 134e5b5f7d..ce3ee7d5b3 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -2,7 +2,6 @@ alter table convoy.configurations add column if not exists cb_sample_rate int not null default 30; -- seconds alter table convoy.configurations add column if not exists cb_error_timeout int not null default 30; -- seconds alter table convoy.configurations add column if not exists cb_failure_threshold int not null default 70; -- percentage -alter table convoy.configurations add column if not exists cb_failure_count int not null default 1; alter table convoy.configurations add column if not exists cb_success_threshold int not null default 1; -- percentage alter table convoy.configurations add column if not exists cb_observability_window int not null default 30; -- minutes alter table convoy.configurations add column if not exists cb_minimum_request_count int not null default 10; @@ -16,7 +15,6 @@ create index if not exists idx_delivery_attempts_event_delivery_id on convoy.del alter table convoy.configurations drop column if exists cb_sample_rate; alter table convoy.configurations drop column if exists cb_error_timeout; alter table convoy.configurations drop column if exists cb_failure_threshold; -alter table convoy.configurations drop column if exists cb_failure_count; alter table convoy.configurations drop column if exists cb_success_threshold; alter table convoy.configurations drop column if exists cb_observability_window; alter table convoy.configurations drop column if exists cb_notification_thresholds; diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index 4b4f22c62f..f42893ba2e 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -128,23 +128,6 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive if breakerErr != nil { return &CircuitBreakerError{Err: breakerErr} } - - // check the circuit breaker state so we can disable the endpoint - cb, breakerErr := circuitBreakerManager.GetCircuitBreaker(ctx, endpoint.UID) - if breakerErr != nil { - return &CircuitBreakerError{Err: breakerErr} - } - - if cb != nil { - if cb.ConsecutiveFailures > circuitBreakerManager.GetConfig().ConsecutiveFailureThreshold { - endpointStatus := datastore.InactiveEndpointStatus - - breakerErr = endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, endpointStatus) - if breakerErr != nil { - log.WithError(breakerErr).Error("failed to deactivate endpoint after failed retry") - } - } - } } err = eventDeliveryRepo.UpdateStatusOfEventDelivery(ctx, project.UID, *eventDelivery, datastore.ProcessingEventStatus) From cace94665cd970e8e48711ced668d1832b9270a6 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 1 Oct 2024 15:18:06 +0200 Subject: [PATCH 37/48] chore: fix tests --- database/postgres/configuration_test.go | 1 - worker/task/process_event_delivery_test.go | 1 - worker/task/process_retry_event_delivery_test.go | 1 - worker/task/retention_policies_test.go | 1 - 4 files changed, 4 deletions(-) diff --git a/database/postgres/configuration_test.go b/database/postgres/configuration_test.go index a83ef554c9..6ad98b8b26 100644 --- a/database/postgres/configuration_test.go +++ b/database/postgres/configuration_test.go @@ -117,7 +117,6 @@ func generateConfig() *datastore.Configuration { SampleRate: 30, ErrorTimeout: 30, FailureThreshold: 10, - FailureCount: 10, SuccessThreshold: 5, ObservabilityWindow: 5, NotificationThresholds: pq.Int64Array{5, 10}, diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 35b8688b12..742985e0d1 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -961,7 +961,6 @@ func TestProcessEventDelivery(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, MinimumRequestCount: 10, diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 8bdeb62f95..02c814fc85 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -1068,7 +1068,6 @@ func TestProcessRetryEventDelivery(t *testing.T) { SampleRate: 1, BreakerTimeout: 30, FailureThreshold: 50, - FailureCount: 5, SuccessThreshold: 2, ObservabilityWindow: 5, MinimumRequestCount: 10, diff --git a/worker/task/retention_policies_test.go b/worker/task/retention_policies_test.go index 79558e3cf3..614f10b8fb 100644 --- a/worker/task/retention_policies_test.go +++ b/worker/task/retention_policies_test.go @@ -452,7 +452,6 @@ func seedConfiguration(db database.Database) (*datastore.Configuration, error) { SampleRate: 2, ErrorTimeout: 30, FailureThreshold: 10, - FailureCount: 3, SuccessThreshold: 1, ObservabilityWindow: 5, NotificationThresholds: pq.Int64Array{10}, From 9a4b9409a32e6bfe7a905a51d6c9fedcb04c55fc Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 1 Oct 2024 15:30:02 +0200 Subject: [PATCH 38/48] chore: remove break --- cmd/worker/worker.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index c972d5862a..baa394680c 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -294,8 +294,6 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte if innerErr != nil { return innerErr } - - break case cb.TypeDisableResource: breakerErr := endpointRepo.UpdateEndpointStatus(ctx, b.TenantId, endpointId, datastore.InactiveEndpointStatus) if breakerErr != nil { @@ -313,7 +311,6 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte if innerErr != nil { return innerErr } - break default: return fmt.Errorf("unsupported circuit breaker notification type: %s", n) } From 6059d92ae3670c594f883f81d08d2e43df0cc307 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 2 Oct 2024 15:41:02 +0200 Subject: [PATCH 39/48] feat: add failure rates to endpoint api response --- api/handlers/endpoint.go | 31 +++++++++++++++++++ api/models/models.go | 2 +- api/server_suite_test.go | 3 ++ api/types/types.go | 2 ++ cmd/agent/agent.go | 1 + cmd/hooks/hooks.go | 6 ++++ cmd/server/server.go | 1 + datastore/models.go | 5 +-- internal/pkg/cli/cli.go | 2 ++ pkg/circuit_breaker/config_test.go | 15 +++++++++ pkg/clock/clock.go | 2 +- .../src/app/models/endpoint.model.ts | 1 + .../endpoints/endpoints.component.html | 13 +++++++- .../project/endpoints/endpoints.component.ts | 29 +++++++++++++++-- .../project/endpoints/endpoints.service.ts | 16 ++++++++++ 15 files changed, 122 insertions(+), 7 deletions(-) diff --git a/api/handlers/endpoint.go b/api/handlers/endpoint.go index 28103ff580..0b01afd456 100644 --- a/api/handlers/endpoint.go +++ b/api/handlers/endpoint.go @@ -3,6 +3,9 @@ package handlers import ( "context" "encoding/json" + "fmt" + "github.com/frain-dev/convoy/pkg/circuit_breaker" + "github.com/frain-dev/convoy/pkg/msgpack" "net/http" "github.com/frain-dev/convoy/api/models" @@ -207,9 +210,37 @@ func (h *Handler) GetEndpoints(w http.ResponseWriter, r *http.Request) { return } + // fetch keys from redis and mutate endpoints slice + keys := make([]string, len(endpoints)) + for i := 0; i < len(endpoints); i++ { + keys[i] = fmt.Sprintf("breaker:%s", endpoints[i].UID) + } + + cbs, err := h.A.Redis.MGet(r.Context(), keys...).Result() + if err != nil { + _ = render.Render(w, r, util.NewServiceErrResponse(err)) + return + } + + for i := 0; i < len(cbs); i++ { + if cbs[i] != nil { + str, ok := cbs[i].(string) + if ok { + var c circuit_breaker.CircuitBreaker + asBytes := []byte(str) + innerErr := msgpack.DecodeMsgPack(asBytes, &c) + if innerErr != nil { + continue + } + endpoints[i].FailureRate = c.FailureRate + } + } + } + resp := models.NewListResponse(endpoints, func(endpoint datastore.Endpoint) models.EndpointResponse { return models.EndpointResponse{Endpoint: &endpoint} }) + serverResponse := util.NewServerResponse( "Endpoints fetched successfully", models.PagedResponse{Content: &resp, Pagination: &paginationData}, http.StatusOK) diff --git a/api/models/models.go b/api/models/models.go index fa225e0af1..b9604530af 100644 --- a/api/models/models.go +++ b/api/models/models.go @@ -154,7 +154,7 @@ type PortalLinkResponse struct { DeletedAt null.Time `json:"deleted_at,omitempty"` } -// NewListResponse is generic function for looping over +// NewListResponse is a generic function for looping over // a slice of type M and returning a slice of type T func NewListResponse[T, M any](items []M, fn func(item M) T) []T { results := make([]T, 0) diff --git a/api/server_suite_test.go b/api/server_suite_test.go index 669fbba395..ddb2ee9ac6 100644 --- a/api/server_suite_test.go +++ b/api/server_suite_test.go @@ -128,10 +128,13 @@ func buildServer() *ApplicationHandler { noopCache := ncache.NewNoopCache() r, _ := rlimiter.NewRedisLimiter(cfg.Redis.BuildDsn()) + rd, _ := rdb.NewClient(cfg.Redis.BuildDsn()) + ah, _ := NewApplicationHandler( &types.APIOptions{ DB: db, Queue: newQueue, + Redis: rd.Client(), Logger: logger, Cache: noopCache, Rate: r, diff --git a/api/types/types.go b/api/types/types.go index 6b706f2f14..defd48861d 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -9,6 +9,7 @@ import ( "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/queue" + "github.com/redis/go-redis/v9" ) type ContextKey string @@ -16,6 +17,7 @@ type ContextKey string type APIOptions struct { FFlag *fflag.FFlag DB database.Database + Redis redis.UniversalClient Queue queue.Queuer Logger log.StdLogger Cache cache.Cache diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index a73574d7e9..e509ec132a 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -160,6 +160,7 @@ func startServerComponent(_ context.Context, a *cli.App) error { Logger: lo, Cache: a.Cache, Rate: a.Rate, + Redis: a.Redis, Licenser: a.Licenser, }) if err != nil { diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index c9bc5fe82b..53d1e23534 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -123,6 +123,11 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ lo := log.NewLogger(os.Stdout) + rd, err := rdb.NewClient(cfg.Redis.BuildDsn()) + if err != nil { + return err + } + ca, err = cache.NewCache(cfg.Redis) if err != nil { return err @@ -156,6 +161,7 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ } } + app.Redis = rd.Client() app.DB = postgresDB app.Queue = q app.Logger = lo diff --git a/cmd/server/server.go b/cmd/server/server.go index bca0336118..edfe0c7a51 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -130,6 +130,7 @@ func startConvoyServer(a *cli.App) error { DB: a.DB, Queue: a.Queue, Logger: lo, + Redis: a.Redis, Cache: a.Cache, Rate: a.Rate, Licenser: a.Licenser, diff --git a/datastore/models.go b/datastore/models.go index 168b0bc86b..6baaef7733 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -417,8 +417,9 @@ type Endpoint struct { Events int64 `json:"events,omitempty" db:"event_count"` Authentication *EndpointAuthentication `json:"authentication" db:"authentication"` - RateLimit int `json:"rate_limit" db:"rate_limit"` - RateLimitDuration uint64 `json:"rate_limit_duration" db:"rate_limit_duration"` + RateLimit int `json:"rate_limit" db:"rate_limit"` + RateLimitDuration uint64 `json:"rate_limit_duration" db:"rate_limit_duration"` + FailureRate float64 `json:"failure_rate" db:"-"` CreatedAt time.Time `json:"created_at,omitempty" db:"created_at,omitempty" swaggertype:"string"` UpdatedAt time.Time `json:"updated_at,omitempty" db:"updated_at,omitempty" swaggertype:"string"` diff --git a/internal/pkg/cli/cli.go b/internal/pkg/cli/cli.go index 31e1894c2a..0a8aeadf40 100644 --- a/internal/pkg/cli/cli.go +++ b/internal/pkg/cli/cli.go @@ -2,6 +2,7 @@ package cli import ( "context" + "github.com/redis/go-redis/v9" "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" @@ -18,6 +19,7 @@ import ( type App struct { Version string DB database.Database + Redis redis.UniversalClient Queue queue.Queuer Logger log.StdLogger Cache cache.Cache diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index dcdff4113e..52516cc904 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -103,6 +103,21 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { wantErr: true, err: "Notification threshold at index [0] = 0 must be greater than 0", }, + { + name: "Invalid NotificationThresholds", + config: CircuitBreakerConfig{ + SampleRate: 1, + BreakerTimeout: 30, + FailureThreshold: 5, + MinimumRequestCount: 10, + SuccessThreshold: 2, + ObservabilityWindow: 5, + ConsecutiveFailureThreshold: 2, + NotificationThresholds: [3]uint64{1, 2}, + }, + wantErr: true, + err: "Notification threshold at index [0] = 0 must be greater than 0", + }, { name: "Invalid ConsecutiveFailureThreshold", config: CircuitBreakerConfig{ diff --git a/pkg/clock/clock.go b/pkg/clock/clock.go index 26884a1485..ea006dee97 100644 --- a/pkg/clock/clock.go +++ b/pkg/clock/clock.go @@ -28,7 +28,7 @@ func (_ *realTimeClock) Now() time.Time { return time.Now() } // This object is concurrency safe. type SimulatedClock struct { mu *sync.Mutex - t time.Time // guarded by mu + t time.Time } func NewSimulatedClock(t time.Time) *SimulatedClock { diff --git a/web/ui/dashboard/src/app/models/endpoint.model.ts b/web/ui/dashboard/src/app/models/endpoint.model.ts index 111dc7efba..3161fb5a52 100644 --- a/web/ui/dashboard/src/app/models/endpoint.model.ts +++ b/web/ui/dashboard/src/app/models/endpoint.model.ts @@ -14,6 +14,7 @@ export interface ENDPOINT { uid: string; title: string; advanced_signatures: boolean; + failure_rate: number; authentication: { api_key: { header_value: string; header_name: string }; }; diff --git a/web/ui/dashboard/src/app/private/pages/project/endpoints/endpoints.component.html b/web/ui/dashboard/src/app/private/pages/project/endpoints/endpoints.component.html index 575b07087e..66e20be542 100644 --- a/web/ui/dashboard/src/app/private/pages/project/endpoints/endpoints.component.html +++ b/web/ui/dashboard/src/app/private/pages/project/endpoints/endpoints.component.html @@ -86,7 +86,9 @@

    Endpoints

    - + {{ licenseService.hasLicense('CIRCUIT_BREAKING') ? (endpoint.failure_rate | number) : '' }} + +
    + +
  • + +
    • - {{ licenseService.hasLicense('CIRCUIT_BREAKING') ? (endpoint.failure_rate | number) : '' }} - + + {{ endpoint.failure_rate | number }}% + +
    @@ -121,14 +123,14 @@

    Endpoints

    -
  • - -
    • +
  • + +
    • +
    +
    + + \ No newline at end of file diff --git a/internal/email/templates/reset.password.html b/internal/email/templates/reset.password.html index 1a08440e83..2edb63ad8e 100644 --- a/internal/email/templates/reset.password.html +++ b/internal/email/templates/reset.password.html @@ -6,87 +6,16 @@ - + -
    +
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
    + convoy-logo

    Date: Thu, 3 Oct 2024 15:37:12 +0200 Subject: [PATCH 43/48] feat: remove NotificationThresholds --- cmd/hooks/hooks.go | 6 -- cmd/worker/worker.go | 27 +------- config/config.go | 16 ++--- config/config_test.go | 3 - database/postgres/configuration.go | 12 ++-- database/postgres/configuration_test.go | 2 - datastore/models.go | 22 ++---- .../circuit_breaker_manager.go | 15 +---- .../circuit_breaker_manager_test.go | 34 ---------- pkg/circuit_breaker/config.go | 22 ------ pkg/circuit_breaker/config_test.go | 67 +++---------------- sql/1724236900.sql | 2 - worker/task/process_event_delivery_test.go | 1 - .../task/process_retry_event_delivery_test.go | 1 - worker/task/retention_policies_test.go | 2 - 15 files changed, 30 insertions(+), 202 deletions(-) diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index 53d1e23534..7d551dc2da 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/lib/pq" "io" "os" "time" @@ -333,10 +332,6 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat IsRetentionPolicyEnabled: cfg.RetentionPolicy.IsRetentionPolicyEnabled, } - notificationThresholds := pq.Int64Array{} - for i := range cfg.CircuitBreaker.NotificationThresholds { - notificationThresholds = append(notificationThresholds, int64(cfg.CircuitBreaker.NotificationThresholds[i])) - } circuitBreakerConfig := &datastore.CircuitBreakerConfig{ SampleRate: cfg.CircuitBreaker.SampleRate, ErrorTimeout: cfg.CircuitBreaker.ErrorTimeout, @@ -344,7 +339,6 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat SuccessThreshold: cfg.CircuitBreaker.SuccessThreshold, ObservabilityWindow: cfg.CircuitBreaker.ObservabilityWindow, MinimumRequestCount: cfg.CircuitBreaker.MinimumRequestCount, - NotificationThresholds: notificationThresholds, ConsecutiveFailureThreshold: cfg.CircuitBreaker.ConsecutiveFailureThreshold, } diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index baa394680c..9e343befc0 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/frain-dev/convoy/datastore" - "github.com/frain-dev/convoy/internal/notifications" "github.com/frain-dev/convoy/internal/pkg/fflag" "net/http" "strings" @@ -282,35 +281,11 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte } switch n { - case cb.TypeTriggerThreshold: - innerErr := notifications.SendEndpointNotification(ctx, - endpoint, project, - endpoint.Status, - q, true, - fmt.Sprintf("%+d percent of the events send to endpoint (%s) in project (%s) have failed", - c.NotificationThresholds[b.NotificationsSent], endpoint.Name, project.Name), - fmt.Sprintf("circuit breaker state for %s is %v", endpoint.Name, b.State), - 429) - if innerErr != nil { - return innerErr - } case cb.TypeDisableResource: - breakerErr := endpointRepo.UpdateEndpointStatus(ctx, b.TenantId, endpointId, datastore.InactiveEndpointStatus) + breakerErr := endpointRepo.UpdateEndpointStatus(ctx, project.UID, endpoint.UID, datastore.InactiveEndpointStatus) if breakerErr != nil { return breakerErr } - - innerErr := notifications.SendEndpointNotification(ctx, - endpoint, project, - endpoint.Status, - q, true, - fmt.Sprintf("Endpoint (%s)'s circuit breaker in the project (%s) has tripped %d times; the endpoint has been de-activated", - endpoint.Name, project.Name, c.ConsecutiveFailureThreshold), - fmt.Sprintf("circuit breaker state for endpoint (%s) is %v", endpoint.Name, b.State), - 429) - if innerErr != nil { - return innerErr - } default: return fmt.Errorf("unsupported circuit breaker notification type: %s", n) } diff --git a/config/config.go b/config/config.go index 9bd66b4f68..537daac4a8 100644 --- a/config/config.go +++ b/config/config.go @@ -81,7 +81,6 @@ var DefaultConfiguration = Configuration{ SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Auth: AuthConfiguration{ @@ -272,14 +271,13 @@ type RetentionPolicyConfiguration struct { } type CircuitBreakerConfiguration struct { - SampleRate uint64 `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` - ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` - FailureThreshold uint64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` - SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` - MinimumRequestCount uint64 `json:"minimum_request_count" envconfig:"CONVOY_MINIMUM_REQUEST_COUNT"` - ObservabilityWindow uint64 `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` - NotificationThresholds [3]uint64 `json:"notification_thresholds" envconfig:"CONVOY_CIRCUIT_BREAKER_NOTIFICATION_THRESHOLDS"` - ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` + SampleRate uint64 `json:"sample_rate" envconfig:"CONVOY_CIRCUIT_BREAKER_SAMPLE_RATE"` + ErrorTimeout uint64 `json:"error_timeout" envconfig:"CONVOY_CIRCUIT_BREAKER_ERROR_TIMEOUT"` + FailureThreshold uint64 `json:"failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_FAILURE_THRESHOLD"` + SuccessThreshold uint64 `json:"success_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_SUCCESS_THRESHOLD"` + MinimumRequestCount uint64 `json:"minimum_request_count" envconfig:"CONVOY_MINIMUM_REQUEST_COUNT"` + ObservabilityWindow uint64 `json:"observability_window" envconfig:"CONVOY_CIRCUIT_BREAKER_OBSERVABILITY_WINDOW"` + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" envconfig:"CONVOY_CIRCUIT_BREAKER_CONSECUTIVE_FAILURE_THRESHOLD"` } type AnalyticsConfiguration struct { diff --git a/config/config_test.go b/config/config_test.go index e99e01b77b..9e4b3ee03c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -132,7 +132,6 @@ func TestLoadConfig(t *testing.T) { SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Server: ServerConfiguration{ @@ -213,7 +212,6 @@ func TestLoadConfig(t *testing.T) { SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Redis: RedisConfiguration{ @@ -289,7 +287,6 @@ func TestLoadConfig(t *testing.T) { SuccessThreshold: 5, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, }, Database: DatabaseConfiguration{ diff --git a/database/postgres/configuration.go b/database/postgres/configuration.go index d945a3796b..d369b8c1f8 100644 --- a/database/postgres/configuration.go +++ b/database/postgres/configuration.go @@ -22,10 +22,10 @@ const ( retention_policy_policy, retention_policy_enabled, cb_sample_rate,cb_error_timeout, cb_failure_threshold, cb_success_threshold, - cb_observability_window, cb_notification_thresholds, + cb_observability_window, cb_consecutive_failure_threshold, cb_minimum_request_count ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22); + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21); ` fetchConfiguration = ` @@ -50,7 +50,6 @@ const ( cb_success_threshold AS "circuit_breaker.success_threshold", cb_observability_window AS "circuit_breaker.observability_window", cb_minimum_request_count as "circuit_breaker.minimum_request_count", - cb_notification_thresholds::INTEGER[] AS "circuit_breaker.notification_thresholds", cb_consecutive_failure_threshold AS "circuit_breaker.consecutive_failure_threshold", created_at, updated_at, @@ -81,9 +80,8 @@ const ( cb_failure_threshold = $17, cb_success_threshold = $18, cb_observability_window = $19, - cb_notification_thresholds = $20, - cb_consecutive_failure_threshold = $21, - cb_minimum_request_count = $22, + cb_consecutive_failure_threshold = $20, + cb_minimum_request_count = $21, updated_at = NOW() WHERE id = $1 AND deleted_at IS NULL; ` @@ -137,7 +135,6 @@ func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore. cb.FailureThreshold, cb.SuccessThreshold, cb.ObservabilityWindow, - cb.NotificationThresholds, cb.ConsecutiveFailureThreshold, cb.MinimumRequestCount, ) @@ -210,7 +207,6 @@ func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Con cb.FailureThreshold, cb.SuccessThreshold, cb.ObservabilityWindow, - cb.NotificationThresholds, cb.ConsecutiveFailureThreshold, cb.MinimumRequestCount, ) diff --git a/database/postgres/configuration_test.go b/database/postgres/configuration_test.go index 6ad98b8b26..1fc9411a1d 100644 --- a/database/postgres/configuration_test.go +++ b/database/postgres/configuration_test.go @@ -6,7 +6,6 @@ package postgres import ( "context" "errors" - "github.com/lib/pq" "testing" "time" @@ -119,7 +118,6 @@ func generateConfig() *datastore.Configuration { FailureThreshold: 10, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: pq.Int64Array{5, 10}, ConsecutiveFailureThreshold: 10, }, } diff --git a/datastore/models.go b/datastore/models.go index 6baaef7733..468a210bb8 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -325,7 +325,6 @@ var ( FailureThreshold: 70, SuccessThreshold: 5, ObservabilityWindow: 5, - NotificationThresholds: pq.Int64Array{10, 30, 50}, ConsecutiveFailureThreshold: 10, } ) @@ -1357,11 +1356,6 @@ func (c *Configuration) GetCircuitBreakerConfig() CircuitBreakerConfig { } func (c *Configuration) ToCircuitBreakerConfig() *cb.CircuitBreakerConfig { - notificationThresholds := [3]uint64{} - for i := range c.CircuitBreakerConfig.NotificationThresholds { - notificationThresholds[i] = uint64(c.CircuitBreakerConfig.NotificationThresholds[i]) - } - return &cb.CircuitBreakerConfig{ SampleRate: c.CircuitBreakerConfig.SampleRate, BreakerTimeout: c.CircuitBreakerConfig.ErrorTimeout, @@ -1369,7 +1363,6 @@ func (c *Configuration) ToCircuitBreakerConfig() *cb.CircuitBreakerConfig { SuccessThreshold: c.CircuitBreakerConfig.SuccessThreshold, ObservabilityWindow: c.CircuitBreakerConfig.ObservabilityWindow, MinimumRequestCount: c.CircuitBreakerConfig.MinimumRequestCount, - NotificationThresholds: notificationThresholds, ConsecutiveFailureThreshold: c.CircuitBreakerConfig.ConsecutiveFailureThreshold, } } @@ -1402,14 +1395,13 @@ type OnPremStorage struct { } type CircuitBreakerConfig struct { - SampleRate uint64 `json:"sample_rate" db:"sample_rate"` - ErrorTimeout uint64 `json:"error_timeout" db:"error_timeout"` - FailureThreshold uint64 `json:"failure_threshold" db:"failure_threshold"` - SuccessThreshold uint64 `json:"success_threshold" db:"success_threshold"` - ObservabilityWindow uint64 `json:"observability_window" db:"observability_window"` - MinimumRequestCount uint64 `json:"minimum_request_count" db:"minimum_request_count"` - NotificationThresholds pq.Int64Array `json:"notification_thresholds" db:"notification_thresholds"` - ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" db:"consecutive_failure_threshold"` + SampleRate uint64 `json:"sample_rate" db:"sample_rate"` + ErrorTimeout uint64 `json:"error_timeout" db:"error_timeout"` + FailureThreshold uint64 `json:"failure_threshold" db:"failure_threshold"` + SuccessThreshold uint64 `json:"success_threshold" db:"success_threshold"` + ObservabilityWindow uint64 `json:"observability_window" db:"observability_window"` + MinimumRequestCount uint64 `json:"minimum_request_count" db:"minimum_request_count"` + ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold" db:"consecutive_failure_threshold"` } type OrganisationMember struct { diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 2c70945e8f..c086936bdf 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -52,8 +52,7 @@ const ( ) const ( - TypeDisableResource NotificationType = "disable" - TypeTriggerThreshold NotificationType = "trigger" + TypeDisableResource NotificationType = "disable" ) func (s State) String() string { @@ -209,7 +208,6 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma breaker.TotalSuccesses = result.Successes breaker.Requests = breaker.TotalSuccesses + breaker.TotalFailures - prevFailureRate := breaker.FailureRate if breaker.Requests == 0 { breaker.FailureRate = 0 breaker.SuccessRate = 0 @@ -232,17 +230,6 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma // send notifications for each circuit breaker if cb.notificationFn != nil { - if prevFailureRate < breaker.FailureRate && breaker.NotificationsSent < 3 { - if breaker.FailureRate >= float64(cb.config.NotificationThresholds[breaker.NotificationsSent]) { - innerErr := cb.notificationFn(TypeTriggerThreshold, *cb.config, breaker) - if innerErr != nil { - log.WithError(innerErr).Errorf("[circuit breaker] failed to execute threshold notification function") - } - log.Debugf("[circuit breaker] executed threshold notification function at %v", cb.config.NotificationThresholds[breaker.NotificationsSent]) - breaker.NotificationsSent++ - } - } - if breaker.ConsecutiveFailures >= cb.GetConfig().ConsecutiveFailureThreshold { innerErr := cb.notificationFn(TypeDisableResource, *cb.config, breaker) if innerErr != nil { diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index 90e16fbbda..bb124fa4b3 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -34,12 +34,6 @@ func pollResult(t *testing.T, key string, failureCount, successCount uint64) map } } -type notificationTriggered struct { - Rate float64 - Sent uint64 - State State -} - func TestCircuitBreakerManager(t *testing.T) { ctx := context.Background() @@ -65,23 +59,13 @@ func TestCircuitBreakerManager(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } - var triggered []notificationTriggered b, err := NewCircuitBreakerManager( ClockOption(testClock), StoreOption(store), ConfigOption(c), - NotificationFunctionOption(func(n NotificationType, c CircuitBreakerConfig, b CircuitBreaker) error { - triggered = append(triggered, notificationTriggered{ - State: b.State, - Rate: b.FailureRate, - Sent: b.NotificationsSent, - }) - return nil - }), ) require.NoError(t, err) @@ -102,11 +86,6 @@ func TestCircuitBreakerManager(t *testing.T) { testClock.AdvanceTime(time.Minute) } - require.Len(t, triggered, 3) - require.Equal(t, triggered[2].Sent, uint64(2)) - require.Equal(t, triggered[2].State, StateOpen) - require.Equal(t, int64(triggered[2].Rate), int64(90)) - breaker, innerErr := b.GetCircuitBreakerWithError(ctx, endpointId) require.NoError(t, innerErr) @@ -138,7 +117,6 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -193,7 +171,6 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -259,7 +236,6 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 3, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -312,7 +288,6 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { SuccessThreshold: 10, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 30, 50}, ConsecutiveFailureThreshold: 10, } b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) @@ -358,7 +333,6 @@ func TestCircuitBreakerManager_Config(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -415,7 +389,6 @@ func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -456,7 +429,6 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -502,7 +474,6 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -560,7 +531,6 @@ func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -617,7 +587,6 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -696,7 +665,6 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -745,7 +713,6 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } @@ -816,7 +783,6 @@ func TestCircuitBreakerManager_Start(t *testing.T) { SuccessThreshold: 10, MinimumRequestCount: 10, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } diff --git a/pkg/circuit_breaker/config.go b/pkg/circuit_breaker/config.go index 5ea836c45c..ecb67a9a2b 100644 --- a/pkg/circuit_breaker/config.go +++ b/pkg/circuit_breaker/config.go @@ -31,9 +31,6 @@ type CircuitBreakerConfig struct { // polled when determining the number successful and failed requests ObservabilityWindow uint64 `json:"observability_window"` - // NotificationThresholds These are the error thresholds after which we will send out notifications. - NotificationThresholds [3]uint64 `json:"notification_thresholds"` - // ConsecutiveFailureThreshold determines when we ultimately disable the endpoint. // E.g., after 10 consecutive transitions from half-open → open we should disable it. ConsecutiveFailureThreshold uint64 `json:"consecutive_failure_threshold"` @@ -78,25 +75,6 @@ func (c *CircuitBreakerConfig) Validate() error { errs.WriteString("; ") } - for i := 0; i < len(c.NotificationThresholds); i++ { - if c.NotificationThresholds[i] == 0 { - errs.WriteString(fmt.Sprintf("Notification threshold at index [%d] = %d must be greater than 0", i, c.NotificationThresholds[i])) - errs.WriteString("; ") - } - - if c.NotificationThresholds[i] > c.FailureThreshold { - errs.WriteString(fmt.Sprintf("Notification threshold at index [%d] = %d must be less than the failure threshold: %d", i, c.NotificationThresholds[i], c.FailureThreshold)) - errs.WriteString("; ") - } - } - - for i := 0; i < len(c.NotificationThresholds)-1; i++ { - if c.NotificationThresholds[i] >= c.NotificationThresholds[i+1] { - errs.WriteString("NotificationThresholds should be in ascending order") - errs.WriteString("; ") - } - } - if c.ConsecutiveFailureThreshold == 0 { errs.WriteString("ConsecutiveFailureThreshold must be greater than 0") errs.WriteString("; ") diff --git a/pkg/circuit_breaker/config_test.go b/pkg/circuit_breaker/config_test.go index 52516cc904..f42b0c6bb9 100644 --- a/pkg/circuit_breaker/config_test.go +++ b/pkg/circuit_breaker/config_test.go @@ -20,7 +20,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { FailureThreshold: 50, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, MinimumRequestCount: 10, }, @@ -67,12 +66,11 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { { name: "Invalid ObservabilityWindow", config: CircuitBreakerConfig{ - SampleRate: 1, - BreakerTimeout: 30, - FailureThreshold: 50, - SuccessThreshold: 2, - ObservabilityWindow: 0, - NotificationThresholds: [3]uint64{10, 20, 30}, + SampleRate: 1, + BreakerTimeout: 30, + FailureThreshold: 50, + SuccessThreshold: 2, + ObservabilityWindow: 0, }, wantErr: true, err: "ObservabilityWindow must be greater than 0", @@ -80,44 +78,15 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { { name: "ObservabilityWindow should be greater than sample rate", config: CircuitBreakerConfig{ - SampleRate: 200, - BreakerTimeout: 30, - FailureThreshold: 50, - SuccessThreshold: 2, - ObservabilityWindow: 1, - NotificationThresholds: [3]uint64{10, 20, 30}, + SampleRate: 200, + BreakerTimeout: 30, + FailureThreshold: 50, + SuccessThreshold: 2, + ObservabilityWindow: 1, }, wantErr: true, err: "ObservabilityWindow must be greater than the SampleRate", }, - { - name: "Invalid NotificationThresholds", - config: CircuitBreakerConfig{ - SampleRate: 1, - BreakerTimeout: 30, - FailureThreshold: 5, - SuccessThreshold: 2, - ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{}, - }, - wantErr: true, - err: "Notification threshold at index [0] = 0 must be greater than 0", - }, - { - name: "Invalid NotificationThresholds", - config: CircuitBreakerConfig{ - SampleRate: 1, - BreakerTimeout: 30, - FailureThreshold: 5, - MinimumRequestCount: 10, - SuccessThreshold: 2, - ObservabilityWindow: 5, - ConsecutiveFailureThreshold: 2, - NotificationThresholds: [3]uint64{1, 2}, - }, - wantErr: true, - err: "Notification threshold at index [0] = 0 must be greater than 0", - }, { name: "Invalid ConsecutiveFailureThreshold", config: CircuitBreakerConfig{ @@ -126,26 +95,11 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { FailureThreshold: 50, SuccessThreshold: 2, ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 0, }, wantErr: true, err: "ConsecutiveFailureThreshold must be greater than 0", }, - { - name: "NotificationThreshold larger than FailureThreshold", - config: CircuitBreakerConfig{ - SampleRate: 1, - BreakerTimeout: 30, - FailureThreshold: 30, - SuccessThreshold: 2, - ObservabilityWindow: 5, - NotificationThresholds: [3]uint64{30, 50, 60}, - ConsecutiveFailureThreshold: 1, - }, - wantErr: true, - err: "Notification threshold at index [2] = 60 must be less than the failure threshold", - }, { name: "Invalid MinimumRequestCount", config: CircuitBreakerConfig{ @@ -155,7 +109,6 @@ func TestCircuitBreakerConfig_Validate(t *testing.T) { SuccessThreshold: 2, ObservabilityWindow: 5, MinimumRequestCount: 5, - NotificationThresholds: [3]uint64{30, 50, 60}, ConsecutiveFailureThreshold: 1, }, wantErr: true, diff --git a/sql/1724236900.sql b/sql/1724236900.sql index ce3ee7d5b3..aad794430a 100644 --- a/sql/1724236900.sql +++ b/sql/1724236900.sql @@ -5,7 +5,6 @@ alter table convoy.configurations add column if not exists cb_failure_threshold alter table convoy.configurations add column if not exists cb_success_threshold int not null default 1; -- percentage alter table convoy.configurations add column if not exists cb_observability_window int not null default 30; -- minutes alter table convoy.configurations add column if not exists cb_minimum_request_count int not null default 10; -alter table convoy.configurations add column if not exists cb_notification_thresholds int[] not null default ARRAY[10, 30, 50]; alter table convoy.configurations add column if not exists cb_consecutive_failure_threshold int not null default 10; create index if not exists idx_delivery_attempts_created_at on convoy.delivery_attempts (created_at); create index if not exists idx_delivery_attempts_event_delivery_id_created_at on convoy.delivery_attempts (event_delivery_id, created_at); @@ -17,7 +16,6 @@ alter table convoy.configurations drop column if exists cb_error_timeout; alter table convoy.configurations drop column if exists cb_failure_threshold; alter table convoy.configurations drop column if exists cb_success_threshold; alter table convoy.configurations drop column if exists cb_observability_window; -alter table convoy.configurations drop column if exists cb_notification_thresholds; alter table convoy.configurations drop column if exists cb_consecutive_failure_threshold; drop index if exists convoy.idx_delivery_attempts_created_at; drop index if exists convoy.idx_delivery_attempts_event_delivery_id_created_at; diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 742985e0d1..185c675189 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -964,7 +964,6 @@ func TestProcessEventDelivery(t *testing.T) { SuccessThreshold: 2, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 02c814fc85..265c6a806f 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -1071,7 +1071,6 @@ func TestProcessRetryEventDelivery(t *testing.T) { SuccessThreshold: 2, ObservabilityWindow: 5, MinimumRequestCount: 10, - NotificationThresholds: [3]uint64{10, 20, 30}, ConsecutiveFailureThreshold: 3, } diff --git a/worker/task/retention_policies_test.go b/worker/task/retention_policies_test.go index 614f10b8fb..1a67ba17c4 100644 --- a/worker/task/retention_policies_test.go +++ b/worker/task/retention_policies_test.go @@ -3,7 +3,6 @@ package task import ( "context" "fmt" - "github.com/lib/pq" "os" "testing" "time" @@ -454,7 +453,6 @@ func seedConfiguration(db database.Database) (*datastore.Configuration, error) { FailureThreshold: 10, SuccessThreshold: 1, ObservabilityWindow: 5, - NotificationThresholds: pq.Int64Array{10}, ConsecutiveFailureThreshold: 10, }, } From 276743aaf2390a5285262eb42ef42a3c8ec491db Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 4 Oct 2024 17:03:34 +0200 Subject: [PATCH 44/48] feat: fix edge case; use logger with log level --- api/handlers/endpoint.go | 22 ++++++++ cmd/worker/worker.go | 1 + database/postgres/delivery_attempts.go | 16 +++++- internal/pkg/pubsub/ingest.go | 2 - pkg/circuit_breaker/circuit_breaker.go | 51 ++++++++++++++--- .../circuit_breaker_manager.go | 56 +++++++++++++------ .../circuit_breaker_manager_test.go | 28 ++++++++-- pkg/circuit_breaker/circuit_breaker_test.go | 2 +- worker/task/process_event_delivery_test.go | 3 + .../task/process_retry_event_delivery_test.go | 3 + 10 files changed, 146 insertions(+), 38 deletions(-) diff --git a/api/handlers/endpoint.go b/api/handlers/endpoint.go index 442a643c67..8ce30e3c80 100644 --- a/api/handlers/endpoint.go +++ b/api/handlers/endpoint.go @@ -7,6 +7,7 @@ import ( "github.com/frain-dev/convoy/pkg/circuit_breaker" "github.com/frain-dev/convoy/pkg/msgpack" "net/http" + "time" "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/database/postgres" @@ -522,6 +523,27 @@ func (h *Handler) ActivateEndpoint(w http.ResponseWriter, r *http.Request) { return } + cbs, err := h.A.Redis.Get(r.Context(), fmt.Sprintf("breaker:%s", endpoint.UID)).Result() + if err != nil { + h.A.Logger.WithError(err).Error("failed to find circuit breaker") + } + + if len(cbs) > 0 { + var c *circuit_breaker.CircuitBreaker + asBytes := []byte(cbs) + innerErr := msgpack.DecodeMsgPack(asBytes, &c) + if innerErr != nil { + h.A.Logger.WithError(innerErr).Error("failed to decode circuit breaker") + } else { + c.ResetCircuitBreaker(time.Now()) + b, msgPackErr := msgpack.EncodeMsgPack(c) + if msgPackErr != nil { + h.A.Logger.WithError(msgPackErr).Error("failed to encode circuit breaker") + } + h.A.Redis.Set(r.Context(), fmt.Sprintf("breaker:%s", endpoint.UID), b, time.Minute*5) + } + } + resp := &models.EndpointResponse{Endpoint: endpoint} serverResponse := util.NewServerResponse("endpoint status successfully activated", resp, http.StatusAccepted) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 9e343befc0..547c28b24b 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -268,6 +268,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte cb.ConfigOption(configuration.ToCircuitBreakerConfig()), cb.StoreOption(cb.NewRedisStore(rd.Client(), clock.NewRealClock())), cb.ClockOption(clock.NewRealClock()), + cb.LoggerOption(lo), cb.NotificationFunctionOption(func(n cb.NotificationType, c cb.CircuitBreakerConfig, b cb.CircuitBreaker) error { endpointId := strings.Split(b.Key, ":")[1] project, funcErr := projectRepo.FetchProjectByID(ctx, b.TenantId) diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index 6e8bea8800..d28a6887ac 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/circuit_breaker" @@ -141,7 +142,8 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo COUNT(CASE WHEN status = false THEN 1 END) AS failures, COUNT(CASE WHEN status = true THEN 1 END) AS successes FROM convoy.delivery_attempts - WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) group by endpoint_id, project_id; + WHERE created_at >= NOW() - MAKE_INTERVAL(mins := $1) + group by endpoint_id, project_id; ` rows, err := d.db.QueryxContext(ctx, query, lookBackDuration) @@ -166,12 +168,20 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo COUNT(CASE WHEN status = false THEN 1 END) AS failures, COUNT(CASE WHEN status = true THEN 1 END) AS successes FROM convoy.delivery_attempts - WHERE endpoint_id = $1 AND created_at >= $2 group by endpoint_id, project_id; + WHERE endpoint_id = '%s' AND created_at >= TIMESTAMP '%s' AT TIME ZONE 'UTC' + group by endpoint_id, project_id; ` + fmt.Printf("cb: %+v\n", resetTimes) + + customFormat := "2006-01-02 15:04:05" for k, t := range resetTimes { + // remove the old key so it doesn't pollute the results + delete(resultsMap, k) + qq := fmt.Sprintf(query2, k, t.Format(customFormat)) + var rowValue circuit_breaker.PollResult - err = d.db.QueryRowxContext(ctx, query2, k, t).StructScan(&rowValue) + err = d.db.QueryRowxContext(ctx, qq).StructScan(&rowValue) if err != nil { if errors.Is(err, sql.ErrNoRows) { continue diff --git a/internal/pkg/pubsub/ingest.go b/internal/pkg/pubsub/ingest.go index b4ba25723a..0e9d86237c 100644 --- a/internal/pkg/pubsub/ingest.go +++ b/internal/pkg/pubsub/ingest.go @@ -95,8 +95,6 @@ func (i *Ingest) getSourceKeys() []memorystore.Key { } func (i *Ingest) run() error { - i.log.Info("refreshing runner...", len(i.sources)) - // cancel all stale/outdated source runners. staleRows := memorystore.Difference(i.getSourceKeys(), i.table.GetKeys()) for _, key := range staleRows { diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index 426f45e955..b261f09fec 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -30,13 +30,16 @@ type CircuitBreaker struct { ConsecutiveFailures uint64 `json:"consecutive_failures"` // Number of notifications (maximum of 3) sent in the observability window NotificationsSent uint64 `json:"notifications_sent"` + + logger *log.Logger } -func NewCircuitBreaker(key string, tenantId string) *CircuitBreaker { +func NewCircuitBreaker(key string, tenantId string, logger *log.Logger) *CircuitBreaker { return &CircuitBreaker{ Key: key, TenantId: tenantId, State: StateClosed, + logger: logger, NotificationsSent: 0, } } @@ -44,31 +47,61 @@ func NewCircuitBreaker(key string, tenantId string) *CircuitBreaker { func (b *CircuitBreaker) String() (s string) { bytes, err := msgpack.EncodeMsgPack(b) if err != nil { - log.WithError(err).Error("[circuit breaker] failed to encode circuit breaker") + if b.logger != nil { + b.logger.WithError(err).Error("[circuit breaker] failed to encode circuit breaker") + } return "" } return string(bytes) } +func (b *CircuitBreaker) asKeyValue() map[string]interface{} { + kv := map[string]interface{}{} + kv["key"] = b.Key + kv["tenant_id"] = b.TenantId + kv["state"] = b.State.String() + kv["requests"] = b.Requests + kv["failure_rate"] = b.FailureRate + kv["success_rate"] = b.SuccessRate + kv["will_reset_at"] = b.WillResetAt + kv["total_failures"] = b.TotalFailures + kv["total_successes"] = b.TotalSuccesses + kv["consecutive_failures"] = b.ConsecutiveFailures + kv["notifications_sent"] = b.NotificationsSent + return kv +} + func (b *CircuitBreaker) tripCircuitBreaker(resetTime time.Time) { b.State = StateOpen b.WillResetAt = resetTime b.ConsecutiveFailures++ - log.Infof("[circuit breaker] circuit breaker transitioned from closed to open.") - log.Debugf("[circuit breaker] circuit breaker state: %+v", b) + if b.logger != nil { + b.logger.Infof("[circuit breaker] circuit breaker transitioned to open.") + b.logger.Debugf("[circuit breaker] circuit breaker state: %+v", b.asKeyValue()) + } } func (b *CircuitBreaker) toHalfOpen() { b.State = StateHalfOpen - log.Infof("[circuit breaker] circuit breaker transitioned from open to half-open") - log.Debugf("[circuit breaker] circuit breaker state: %+v", b) + if b.logger != nil { + b.logger.Infof("[circuit breaker] circuit breaker transitioned from open to half-open") + b.logger.Debugf("[circuit breaker] circuit breaker state: %+v", b.asKeyValue()) + } } -func (b *CircuitBreaker) resetCircuitBreaker() { +func (b *CircuitBreaker) ResetCircuitBreaker(resetTime time.Time) { b.State = StateClosed + b.WillResetAt = resetTime b.NotificationsSent = 0 b.ConsecutiveFailures = 0 - log.Infof("[circuit breaker] circuit breaker transitioned from half-open to closed") - log.Debugf("[circuit breaker] circuit breaker state: %+v", b) + b.FailureRate = 0 + b.SuccessRate = 0 + b.TotalFailures = 0 + b.TotalSuccesses = 0 + b.Requests = 0 + if b.logger != nil { + b.logger.Infof("[circuit breaker] circuit breaker transitioned from half-open to closed") + b.logger.Debugf("[circuit breaker] circuit breaker state: %+v", b.asKeyValue()) + } } diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index c086936bdf..dbeb5493d8 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -36,6 +36,9 @@ var ( // ErrConfigMustNotBeNil is returned when a nil config is passed to NewCircuitBreakerManager ErrConfigMustNotBeNil = errors.New("[circuit breaker] config must not be nil") + // ErrLoggerMustNotBeNil is returned when a nil logger is passed to NewCircuitBreakerManager + ErrLoggerMustNotBeNil = errors.New("[circuit breaker] logger must not be nil") + // ErrNotificationFunctionMustNotBeNil is returned when a nil function is passed to NewCircuitBreakerManager ErrNotificationFunctionMustNotBeNil = errors.New("[circuit breaker] notification function must not be nil") ) @@ -77,6 +80,7 @@ type PollResult struct { type CircuitBreakerManager struct { config *CircuitBreakerConfig + logger *log.Logger clock clock.Clock store CircuitBreakerStore notificationFn func(NotificationType, CircuitBreakerConfig, CircuitBreaker) error @@ -104,6 +108,10 @@ func NewCircuitBreakerManager(options ...CircuitBreakerOption) (*CircuitBreakerM return nil, ErrConfigMustNotBeNil } + if r.logger == nil { + return nil, ErrLoggerMustNotBeNil + } + return r, nil } @@ -144,6 +152,17 @@ func ConfigOption(config *CircuitBreakerConfig) CircuitBreakerOption { } } +func LoggerOption(logger *log.Logger) CircuitBreakerOption { + return func(cb *CircuitBreakerManager) error { + if logger == nil { + return ErrLoggerMustNotBeNil + } + + cb.logger = logger + return nil + } +} + func NotificationFunctionOption(fn func(NotificationType, CircuitBreakerConfig, CircuitBreaker) error) CircuitBreakerOption { return func(cb *CircuitBreakerManager) error { if fn == nil { @@ -177,27 +196,29 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma for i := range res { if res[i] == nil { - circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i]) + circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i], cb.logger) continue } str, ok := res[i].(string) if !ok { - log.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) + cb.logger.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) // the circuit breaker is corrupted, create a new one in its place - circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i]) + circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i], cb.logger) continue } - var c CircuitBreaker + var c *CircuitBreaker asBytes := []byte(str) innerErr := msgpack.DecodeMsgPack(asBytes, &c) if innerErr != nil { return innerErr } - circuitBreakers[keys[i]] = c + c.logger = cb.logger + + circuitBreakers[keys[i]] = *c } for key, breaker := range circuitBreakers { @@ -217,7 +238,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma } if breaker.State == StateHalfOpen && breaker.SuccessRate >= float64(cb.config.SuccessThreshold) { - breaker.resetCircuitBreaker() + breaker.ResetCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && breaker.Requests >= cb.config.MinimumRequestCount { if breaker.FailureRate >= float64(cb.config.FailureThreshold) { breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) @@ -229,13 +250,13 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma } // send notifications for each circuit breaker - if cb.notificationFn != nil { + if cb.notificationFn != nil && breaker.State != StateOpen { if breaker.ConsecutiveFailures >= cb.GetConfig().ConsecutiveFailureThreshold { innerErr := cb.notificationFn(TypeDisableResource, *cb.config, breaker) if innerErr != nil { - log.WithError(innerErr).Errorf("[circuit breaker] failed to execute disable resource notification function") + cb.logger.WithError(innerErr).Errorf("[circuit breaker] failed to execute disable resource notification function") } - log.Debug("[circuit breaker] executed disable resource notification function") + cb.logger.Debug("[circuit breaker] executed disable resource notification function") } } @@ -243,7 +264,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma } if err = cb.updateCircuitBreakers(ctx, circuitBreakers); err != nil { - log.WithError(err).Error("[circuit breaker] failed to update state") + cb.logger.WithError(err).Error("[circuit breaker] failed to update state") return err } @@ -302,7 +323,7 @@ func (cb *CircuitBreakerManager) getCircuitBreakerError(b CircuitBreaker) error case StateOpen: return ErrOpenState case StateHalfOpen: - if b.FailureRate > float64(cb.config.FailureThreshold) { + if b.FailureRate > float64(cb.config.FailureThreshold) && b.WillResetAt.After(cb.clock.Now()) { return ErrTooManyRequests } return nil @@ -381,27 +402,28 @@ func (cb *CircuitBreakerManager) GetCircuitBreakerWithError(ctx context.Context, func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { mu, err := cb.store.Lock(ctx, mutexKey) if err != nil { - log.WithError(err).Error("[circuit breaker] failed to acquire lock") + cb.logger.WithError(err).Error("[circuit breaker] failed to acquire lock") return err } defer func() { innerErr := cb.store.Unlock(ctx, mu) if innerErr != nil { - log.WithError(innerErr).Error("[circuit breaker] failed to unlock mutex") + cb.logger.WithError(innerErr).Error("[circuit breaker] failed to unlock mutex") } }() bs, err := cb.loadCircuitBreakers(ctx) if err != nil { - log.WithError(err).Error("[circuit breaker] failed to load circuitBreakers") + cb.logger.WithError(err).Error("[circuit breaker] failed to load circuitBreakers") return err } resetMap := make(map[string]time.Time, len(bs)) for i := range bs { - if bs[i].State == StateClosed && bs[i].WillResetAt.After(time.Time{}) { - resetMap[bs[i].Key] = bs[i].WillResetAt + if bs[i].State != StateOpen && bs[i].WillResetAt.After(time.Time{}) { + k := strings.Split(bs[i].Key, ":")[1] + resetMap[k] = bs[i].WillResetAt } } @@ -436,7 +458,7 @@ func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { return case <-ticker.C: if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { - log.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") + cb.logger.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") } } } diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index bb124fa4b3..42b460e3ed 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -3,6 +3,8 @@ package circuit_breaker import ( "context" "errors" + "github.com/frain-dev/convoy/pkg/log" + "os" "testing" "time" @@ -66,6 +68,7 @@ func TestCircuitBreakerManager(t *testing.T) { ClockOption(testClock), StoreOption(store), ConfigOption(c), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -119,7 +122,7 @@ func TestCircuitBreakerManager_AddNewBreakerMidway(t *testing.T) { ObservabilityWindow: 5, ConsecutiveFailureThreshold: 10, } - b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c), LoggerOption(log.NewLogger(os.Stdout))) require.NoError(t, err) endpoint1 := "endpoint-1" @@ -173,7 +176,7 @@ func TestCircuitBreakerManager_Transitions(t *testing.T) { ObservabilityWindow: 5, ConsecutiveFailureThreshold: 10, } - b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c), LoggerOption(log.NewLogger(os.Stdout))) require.NoError(t, err) endpointId := "endpoint-1" @@ -238,7 +241,7 @@ func TestCircuitBreakerManager_ConsecutiveFailures(t *testing.T) { ObservabilityWindow: 5, ConsecutiveFailureThreshold: 3, } - b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c), LoggerOption(log.NewLogger(os.Stdout))) require.NoError(t, err) endpointId := "endpoint-1" @@ -290,7 +293,7 @@ func TestCircuitBreakerManager_MultipleEndpoints(t *testing.T) { MinimumRequestCount: 10, ConsecutiveFailureThreshold: 10, } - b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c)) + b, err := NewCircuitBreakerManager(ClockOption(testClock), StoreOption(store), ConfigOption(c), LoggerOption(log.NewLogger(os.Stdout))) require.NoError(t, err) endpoint1 := "endpoint-1" @@ -341,6 +344,7 @@ func TestCircuitBreakerManager_Config(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -354,6 +358,7 @@ func TestCircuitBreakerManager_Config(t *testing.T) { _, err := NewCircuitBreakerManager( ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.Error(t, err) @@ -364,6 +369,7 @@ func TestCircuitBreakerManager_Config(t *testing.T) { _, err := NewCircuitBreakerManager( StoreOption(mockStore), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.Error(t, err) @@ -374,6 +380,7 @@ func TestCircuitBreakerManager_Config(t *testing.T) { _, err := NewCircuitBreakerManager( StoreOption(mockStore), ClockOption(mockClock), + LoggerOption(log.NewLogger(os.Stdout)), ) require.Error(t, err) @@ -392,7 +399,8 @@ func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { ConsecutiveFailureThreshold: 3, } - manager := &CircuitBreakerManager{config: config} + c := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + manager := &CircuitBreakerManager{config: config, clock: c} t.Run("Open State", func(t *testing.T) { breaker := CircuitBreaker{State: StateOpen} @@ -401,7 +409,7 @@ func TestCircuitBreakerManager_GetCircuitBreakerError(t *testing.T) { }) t.Run("Half-Open State with Too Many Failures", func(t *testing.T) { - breaker := CircuitBreaker{State: StateHalfOpen, FailureRate: 60} + breaker := CircuitBreaker{State: StateHalfOpen, FailureRate: 60, WillResetAt: time.Date(2020, 1, 1, 0, 1, 0, 0, time.UTC)} err := manager.getCircuitBreakerError(breaker) require.Equal(t, ErrTooManyRequests, err) }) @@ -436,6 +444,7 @@ func TestCircuitBreakerManager_SampleStore(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -481,6 +490,7 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -538,6 +548,7 @@ func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -594,6 +605,7 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -633,6 +645,7 @@ func TestCircuitBreakerManager_CanExecute(t *testing.T) { Key: "test_half_open", State: StateHalfOpen, FailureRate: 60, + WillResetAt: time.Date(2020, 1, 1, 0, 1, 0, 0, time.UTC), } err := manager.store.SetOne(ctx, "breaker:test_half_open", cb, time.Minute) require.NoError(t, err) @@ -672,6 +685,7 @@ func TestCircuitBreakerManager_GetCircuitBreaker(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -720,6 +734,7 @@ func TestCircuitBreakerManager_SampleAndUpdate(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) @@ -790,6 +805,7 @@ func TestCircuitBreakerManager_Start(t *testing.T) { StoreOption(mockStore), ClockOption(mockClock), ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index c206ddbc40..8b3bacb2b0 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -71,7 +71,7 @@ func TestCircuitBreaker_resetCircuitBreaker(t *testing.T) { ConsecutiveFailures: 5, } - cb.resetCircuitBreaker() + cb.ResetCircuitBreaker(time.Now()) require.Equal(t, StateClosed, cb.State) require.Equal(t, uint64(0), cb.ConsecutiveFailures) diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 185c675189..549b3742d7 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/pkg/log" + "os" "testing" "github.com/frain-dev/convoy/internal/pkg/license" @@ -971,6 +973,7 @@ func TestProcessEventDelivery(t *testing.T) { cb.StoreOption(mockStore), cb.ClockOption(mockClock), cb.ConfigOption(breakerConfig), + cb.LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 265c6a806f..d187711a57 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/pkg/log" + "os" "testing" "github.com/frain-dev/convoy/internal/pkg/license" @@ -1078,6 +1080,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { cb.StoreOption(mockStore), cb.ClockOption(mockClock), cb.ConfigOption(breakerConfig), + cb.LoggerOption(log.NewLogger(os.Stdout)), ) require.NoError(t, err) From 9ab0419c881c57caf54e09f16f1d1f5ed08a23bd Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 9 Oct 2024 11:36:21 +0200 Subject: [PATCH 45/48] deduplicate how circuit breakers are loaded from the store --- api/handlers/endpoint.go | 6 +- pkg/circuit_breaker/circuit_breaker.go | 16 +- .../circuit_breaker_manager.go | 36 ++--- .../circuit_breaker_manager_test.go | 72 ++++++++- pkg/circuit_breaker/circuit_breaker_test.go | 143 +++++++++++++++++- 5 files changed, 247 insertions(+), 26 deletions(-) diff --git a/api/handlers/endpoint.go b/api/handlers/endpoint.go index 8ce30e3c80..5c3c7f058b 100644 --- a/api/handlers/endpoint.go +++ b/api/handlers/endpoint.go @@ -529,13 +529,11 @@ func (h *Handler) ActivateEndpoint(w http.ResponseWriter, r *http.Request) { } if len(cbs) > 0 { - var c *circuit_breaker.CircuitBreaker - asBytes := []byte(cbs) - innerErr := msgpack.DecodeMsgPack(asBytes, &c) + c, innerErr := circuit_breaker.NewCircuitBreakerFromStore([]byte(cbs), h.A.Logger.(*log.Logger)) if innerErr != nil { h.A.Logger.WithError(innerErr).Error("failed to decode circuit breaker") } else { - c.ResetCircuitBreaker(time.Now()) + c.Reset(time.Now()) b, msgPackErr := msgpack.EncodeMsgPack(c) if msgPackErr != nil { h.A.Logger.WithError(msgPackErr).Error("failed to encode circuit breaker") diff --git a/pkg/circuit_breaker/circuit_breaker.go b/pkg/circuit_breaker/circuit_breaker.go index b261f09fec..efc8df5666 100644 --- a/pkg/circuit_breaker/circuit_breaker.go +++ b/pkg/circuit_breaker/circuit_breaker.go @@ -44,6 +44,18 @@ func NewCircuitBreaker(key string, tenantId string, logger *log.Logger) *Circuit } } +func NewCircuitBreakerFromStore(b []byte, logger *log.Logger) (*CircuitBreaker, error) { + var c *CircuitBreaker + innerErr := msgpack.DecodeMsgPack(b, &c) + if innerErr != nil { + return nil, innerErr + } + + c.logger = logger + + return c, nil +} + func (b *CircuitBreaker) String() (s string) { bytes, err := msgpack.EncodeMsgPack(b) if err != nil { @@ -72,7 +84,7 @@ func (b *CircuitBreaker) asKeyValue() map[string]interface{} { return kv } -func (b *CircuitBreaker) tripCircuitBreaker(resetTime time.Time) { +func (b *CircuitBreaker) trip(resetTime time.Time) { b.State = StateOpen b.WillResetAt = resetTime b.ConsecutiveFailures++ @@ -90,7 +102,7 @@ func (b *CircuitBreaker) toHalfOpen() { } } -func (b *CircuitBreaker) ResetCircuitBreaker(resetTime time.Time) { +func (b *CircuitBreaker) Reset(resetTime time.Time) { b.State = StateClosed b.WillResetAt = resetTime b.NotificationsSent = 0 diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index dbeb5493d8..d8cfc1803d 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -195,6 +195,7 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma } for i := range res { + // the circuit breaker doesn't exist if res[i] == nil { circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i], cb.logger) continue @@ -202,22 +203,18 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma str, ok := res[i].(string) if !ok { - cb.logger.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) - // the circuit breaker is corrupted, create a new one in its place + cb.logger.Errorf("[circuit breaker] breaker with key (%s) is corrupted, reseting it", keys[i]) circuitBreakers[keys[i]] = *NewCircuitBreaker(keys[i], tenants[i], cb.logger) continue } - var c *CircuitBreaker - asBytes := []byte(str) - innerErr := msgpack.DecodeMsgPack(asBytes, &c) + c, innerErr := NewCircuitBreakerFromStore([]byte(str), cb.logger) if innerErr != nil { - return innerErr + cb.logger.WithError(innerErr).Errorf("[circuit breaker] an error occurred loading circuit breaker (%s) state from the store", keys[i]) + continue } - c.logger = cb.logger - circuitBreakers[keys[i]] = *c } @@ -238,10 +235,10 @@ func (cb *CircuitBreakerManager) sampleStore(ctx context.Context, pollResults ma } if breaker.State == StateHalfOpen && breaker.SuccessRate >= float64(cb.config.SuccessThreshold) { - breaker.ResetCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) + breaker.Reset(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) } else if (breaker.State == StateClosed || breaker.State == StateHalfOpen) && breaker.Requests >= cb.config.MinimumRequestCount { if breaker.FailureRate >= float64(cb.config.FailureThreshold) { - breaker.tripCircuitBreaker(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) + breaker.trip(cb.clock.Now().Add(time.Duration(cb.config.BreakerTimeout) * time.Second)) } } @@ -300,19 +297,24 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir circuitBreakers := make([]CircuitBreaker, len(res)) for i := range res { - c := CircuitBreaker{} switch res[i].(type) { case string: - asBytes := []byte(res[i].(string)) - innerErr := msgpack.DecodeMsgPack(asBytes, &c) + str, ok := res[i].(string) + if !ok { + cb.logger.Errorf("[circuit breaker] breaker with key (%s) is corrupted", keys[i]) + continue + } + + c, innerErr := NewCircuitBreakerFromStore([]byte(str), cb.logger) if innerErr != nil { - return nil, innerErr + cb.logger.WithError(innerErr).Errorf("[circuit breaker] an error occurred loading circuit breaker (%s) state from the store", keys[i]) + continue } - case CircuitBreaker: - c = res[i].(CircuitBreaker) + circuitBreakers[i] = *c + case CircuitBreaker: // only used in tests that use the mockStore + circuitBreakers[i] = res[i].(CircuitBreaker) } - circuitBreakers[i] = c } return circuitBreakers, nil diff --git a/pkg/circuit_breaker/circuit_breaker_manager_test.go b/pkg/circuit_breaker/circuit_breaker_manager_test.go index 42b460e3ed..fbcaa71956 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager_test.go +++ b/pkg/circuit_breaker/circuit_breaker_manager_test.go @@ -531,7 +531,7 @@ func TestCircuitBreakerManager_UpdateCircuitBreakers(t *testing.T) { require.Equal(t, uint64(4), cb2.TotalSuccesses) } -func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { +func TestCircuitBreakerManager_LoadCircuitBreakers_TestStore(t *testing.T) { mockStore := NewTestStore() mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) config := &CircuitBreakerConfig{ @@ -588,6 +588,76 @@ func TestCircuitBreakerManager_LoadCircuitBreakers(t *testing.T) { } } +func TestCircuitBreakerManager_LoadCircuitBreakers_RedisStore(t *testing.T) { + ctx := context.Background() + + re, err := getRedis(t) + require.NoError(t, err) + + mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + store := NewRedisStore(re, mockClock) + + keys, err := re.Keys(ctx, "breaker*").Result() + require.NoError(t, err) + + for i := range keys { + err = re.Del(ctx, keys[i]).Err() + require.NoError(t, err) + } + + config := &CircuitBreakerConfig{ + SampleRate: 1, + BreakerTimeout: 30, + FailureThreshold: 50, + SuccessThreshold: 10, + MinimumRequestCount: 10, + ObservabilityWindow: 5, + ConsecutiveFailureThreshold: 3, + } + + manager, err := NewCircuitBreakerManager( + StoreOption(store), + ClockOption(mockClock), + ConfigOption(config), + LoggerOption(log.NewLogger(os.Stdout)), + ) + require.NoError(t, err) + + breakers := map[string]CircuitBreaker{ + "breaker:test1": { + Key: "test1", + State: StateClosed, + Requests: 10, + TotalFailures: 3, + TotalSuccesses: 7, + }, + "breaker:test2": { + Key: "test2", + State: StateOpen, + Requests: 10, + TotalFailures: 6, + TotalSuccesses: 4, + }, + } + + err = manager.updateCircuitBreakers(ctx, breakers) + require.NoError(t, err) + + loadedBreakers, err := manager.loadCircuitBreakers(ctx) + require.NoError(t, err) + require.Len(t, loadedBreakers, 2) + + // Check if loaded circuit breakers match the original ones + for _, cb := range loadedBreakers { + originalCB, exists := breakers["breaker:"+cb.Key] + require.True(t, exists) + require.Equal(t, originalCB.State, cb.State) + require.Equal(t, originalCB.Requests, cb.Requests) + require.Equal(t, originalCB.TotalFailures, cb.TotalFailures) + require.Equal(t, originalCB.TotalSuccesses, cb.TotalSuccesses) + } +} + func TestCircuitBreakerManager_CanExecute(t *testing.T) { mockStore := NewTestStore() mockClock := clock.NewSimulatedClock(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) diff --git a/pkg/circuit_breaker/circuit_breaker_test.go b/pkg/circuit_breaker/circuit_breaker_test.go index 8b3bacb2b0..5f68917bdc 100644 --- a/pkg/circuit_breaker/circuit_breaker_test.go +++ b/pkg/circuit_breaker/circuit_breaker_test.go @@ -1,8 +1,11 @@ package circuit_breaker import ( + "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/pkg/msgpack" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "os" "testing" "time" ) @@ -48,7 +51,7 @@ func TestCircuitBreaker_tripCircuitBreaker(t *testing.T) { } resetTime := time.Now().Add(30 * time.Second) - cb.tripCircuitBreaker(resetTime) + cb.trip(resetTime) require.Equal(t, StateOpen, cb.State) require.Equal(t, resetTime, cb.WillResetAt) @@ -71,8 +74,144 @@ func TestCircuitBreaker_resetCircuitBreaker(t *testing.T) { ConsecutiveFailures: 5, } - cb.ResetCircuitBreaker(time.Now()) + cb.Reset(time.Now()) require.Equal(t, StateClosed, cb.State) require.Equal(t, uint64(0), cb.ConsecutiveFailures) } + +func TestNewCircuitBreakerFromStore(t *testing.T) { + createValidMsgpack := func() []byte { + cb := &CircuitBreaker{ + Key: "test-key", + TenantId: "tenant-1", + State: StateClosed, + Requests: 10, + FailureRate: 0.2, + SuccessRate: 0.8, + WillResetAt: time.Now().Add(time.Hour), + TotalFailures: 2, + TotalSuccesses: 8, + ConsecutiveFailures: 1, + NotificationsSent: 1, + } + data, err := msgpack.EncodeMsgPack(cb) + if err != nil { + t.Fatalf("Failed to create test data: %v", err) + } + return data + } + + logger := log.NewLogger(os.Stdout) + + tests := []struct { + name string + input []byte + logger *log.Logger + wantErr bool + errContains string + validate func(*testing.T, *CircuitBreaker) + }{ + { + name: "empty input", + input: []byte{}, + logger: logger, + wantErr: true, + errContains: "EOF", + }, + { + name: "invalid CircuitBreaker", + input: []byte{0x1, 0x2, 0x3}, + logger: logger, + wantErr: true, + errContains: "decoding map length", + }, + { + name: "valid CircuitBreaker with logger", + input: createValidMsgpack(), + logger: logger, + wantErr: false, + validate: func(t *testing.T, cb *CircuitBreaker) { + assert.Equal(t, "test-key", cb.Key) + assert.Equal(t, "tenant-1", cb.TenantId) + assert.Equal(t, StateClosed, cb.State) + assert.Equal(t, uint64(10), cb.Requests) + assert.Equal(t, 0.2, cb.FailureRate) + assert.Equal(t, 0.8, cb.SuccessRate) + assert.Equal(t, uint64(2), cb.TotalFailures) + assert.Equal(t, uint64(8), cb.TotalSuccesses) + assert.Equal(t, uint64(1), cb.ConsecutiveFailures) + assert.Equal(t, uint64(1), cb.NotificationsSent) + assert.NotNil(t, cb.logger) + }, + }, + { + name: "valid CircuitBreaker without logger", + input: createValidMsgpack(), + logger: nil, + wantErr: false, + validate: func(t *testing.T, cb *CircuitBreaker) { + assert.Equal(t, "test-key", cb.Key) + assert.Nil(t, cb.logger) + }, + }, + { + name: "CircuitBreaker with different state", + input: func() []byte { + cb := &CircuitBreaker{ + Key: "test-key", + State: StateOpen, + } + data, _ := msgpack.EncodeMsgPack(cb) + return data + }(), + logger: logger, + wantErr: false, + validate: func(t *testing.T, cb *CircuitBreaker) { + assert.Equal(t, StateOpen, cb.State) + }, + }, + { + name: "large numbers test", + input: func() []byte { + cb := &CircuitBreaker{ + Key: "test-key", + Requests: 18446744073709551615, // max uint64 + TotalSuccesses: 18446744073709551615, // max uint64 + ConsecutiveFailures: 18446744073709551615, // max uint64 + } + data, _ := msgpack.EncodeMsgPack(cb) + return data + }(), + logger: logger, + wantErr: false, + validate: func(t *testing.T, cb *CircuitBreaker) { + assert.Equal(t, uint64(18446744073709551615), cb.Requests) + assert.Equal(t, uint64(18446744073709551615), cb.TotalSuccesses) + assert.Equal(t, uint64(18446744073709551615), cb.ConsecutiveFailures) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewCircuitBreakerFromStore(tt.input, tt.logger) + + if tt.wantErr { + assert.Error(t, err) + if len(tt.errContains) > 0 { + assert.Contains(t, err.Error(), tt.errContains) + } + assert.Nil(t, got) + return + } + + assert.NoError(t, err) + assert.NotNil(t, got) + + if tt.validate != nil { + tt.validate(t, got) + } + }) + } +} From 4ef0810999018ce245d25f9431239246bf9c849c Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 9 Oct 2024 16:46:26 +0200 Subject: [PATCH 46/48] feat: fix distributed locking mechanism; add sample latency metric --- cmd/agent/agent.go | 27 ++------------- database/postgres/delivery_attempts.go | 2 -- internal/pkg/metrics/data_plane.go | 2 +- .../circuit_breaker_collector.go | 22 +++++++++--- .../circuit_breaker_manager.go | 34 ++++++++++++++++--- pkg/circuit_breaker/store.go | 10 +++--- worker/task/process_event_delivery.go | 2 +- 7 files changed, 56 insertions(+), 43 deletions(-) diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index e509ec132a..08274b9841 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -25,8 +25,6 @@ import ( func AddAgentCommand(a *cli.App) *cobra.Command { var agentPort uint32 - var ingestPort uint32 - var workerPort uint32 var logLevel string var consumerPoolSize int var interval int @@ -109,9 +107,7 @@ func AddAgentCommand(a *cli.App) *cobra.Command { cmd.Flags().StringVar(&smtpUrl, "smtp-url", "", "SMTP provider URL") cmd.Flags().Uint32Var(&smtpPort, "smtp-port", 0, "SMTP Port") - cmd.Flags().Uint32Var(&agentPort, "agent-port", 0, "Agent port") - cmd.Flags().Uint32Var(&workerPort, "worker-port", 0, "Worker port") - cmd.Flags().Uint32Var(&ingestPort, "ingest-port", 0, "Ingest port") + cmd.Flags().Uint32Var(&agentPort, "port", 0, "Agent port") cmd.Flags().StringVar(&logLevel, "log-level", "", "Log level") cmd.Flags().IntVar(&consumerPoolSize, "consumers", -1, "Size of the consumers pool.") @@ -191,7 +187,7 @@ func buildAgentCliConfiguration(cmd *cobra.Command) (*config.Configuration, erro c := &config.Configuration{} // PORT - port, err := cmd.Flags().GetUint32("agent-port") + port, err := cmd.Flags().GetUint32("port") if err != nil { return nil, err } @@ -200,25 +196,6 @@ func buildAgentCliConfiguration(cmd *cobra.Command) (*config.Configuration, erro c.Server.HTTP.AgentPort = port } - ingestPort, err := cmd.Flags().GetUint32("ingest-port") - if err != nil { - return nil, err - } - - if ingestPort != 0 { - c.Server.HTTP.IngestPort = ingestPort - } - - // CONVOY_WORKER_PORT - workerPort, err := cmd.Flags().GetUint32("worker-port") - if err != nil { - return nil, err - } - - if workerPort != 0 { - c.Server.HTTP.WorkerPort = workerPort - } - logLevel, err := cmd.Flags().GetString("log-level") if err != nil { return nil, err diff --git a/database/postgres/delivery_attempts.go b/database/postgres/delivery_attempts.go index d28a6887ac..f525c0f5e0 100644 --- a/database/postgres/delivery_attempts.go +++ b/database/postgres/delivery_attempts.go @@ -172,8 +172,6 @@ func (d *deliveryAttemptRepo) GetFailureAndSuccessCounts(ctx context.Context, lo group by endpoint_id, project_id; ` - fmt.Printf("cb: %+v\n", resetTimes) - customFormat := "2006-01-02 15:04:05" for k, t := range resetTimes { // remove the old key so it doesn't pollute the results diff --git a/internal/pkg/metrics/data_plane.go b/internal/pkg/metrics/data_plane.go index 72d7113834..96e68b9e08 100644 --- a/internal/pkg/metrics/data_plane.go +++ b/internal/pkg/metrics/data_plane.go @@ -108,7 +108,7 @@ func InitMetrics(licenser license.Licenser) *Metrics { return m } -func (m *Metrics) RecordLatency(ev *datastore.EventDelivery) { +func (m *Metrics) RecordEndToEndLatency(ev *datastore.EventDelivery) { if !m.IsEnabled { return } diff --git a/pkg/circuit_breaker/circuit_breaker_collector.go b/pkg/circuit_breaker/circuit_breaker_collector.go index f76888e875..53d4779e77 100644 --- a/pkg/circuit_breaker/circuit_breaker_collector.go +++ b/pkg/circuit_breaker/circuit_breaker_collector.go @@ -55,10 +55,16 @@ var ( "Number of notifications sent by the circuit breaker", []string{"key", "tenant_id"}, nil, ) + circuitBreakerSampleLatency = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "sample_latency_seconds"), + "Latency of the circuit breaker sampling process in seconds", + nil, nil, + ) ) type Metrics struct { circuitBreakers []CircuitBreaker + SampleLatency time.Duration } func (cb *CircuitBreakerManager) collectMetrics() (*Metrics, error) { @@ -78,6 +84,11 @@ func (cb *CircuitBreakerManager) Describe(ch chan<- *prometheus.Desc) { } func (cb *CircuitBreakerManager) Collect(ch chan<- prometheus.Metric) { + var metrics *Metrics + var err error + now := time.Now() + cachedMetrics = &Metrics{} + if metricsConfig == nil { cfg, err := config.Get() if err != nil { @@ -85,14 +96,12 @@ func (cb *CircuitBreakerManager) Collect(ch chan<- prometheus.Metric) { } metricsConfig = &cfg.Metrics } + if !metricsConfig.IsEnabled { return } - var metrics *Metrics - var err error - now := time.Now() - if cachedMetrics != nil && lastRun.Add(time.Duration(metricsConfig.Prometheus.SampleTime)*time.Second).After(now) { + if lastRun.Add(time.Duration(metricsConfig.Prometheus.SampleTime) * time.Second).After(now) { metrics = cachedMetrics } else { metrics, err = cb.collectMetrics() @@ -160,6 +169,11 @@ func (cb *CircuitBreakerManager) Collect(ch chan<- prometheus.Metric) { metric.Key, metric.TenantId, ) + ch <- prometheus.MustNewConstMetric( + circuitBreakerSampleLatency, + prometheus.GaugeValue, + metrics.SampleLatency.Seconds(), + ) } lastRun = now diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index d8cfc1803d..6e86b8546f 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -314,7 +314,6 @@ func (cb *CircuitBreakerManager) loadCircuitBreakers(ctx context.Context) ([]Cir case CircuitBreaker: // only used in tests that use the mockStore circuitBreakers[i] = res[i].(CircuitBreaker) } - } return circuitBreakers, nil @@ -402,16 +401,41 @@ func (cb *CircuitBreakerManager) GetCircuitBreakerWithError(ctx context.Context, } func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { - mu, err := cb.store.Lock(ctx, mutexKey) + start := time.Now() + stopTime := time.Now().Add(time.Duration(cb.config.SampleRate) * time.Second) + isLeader := true + + mu, err := cb.store.Lock(ctx, mutexKey, cb.config.SampleRate) if err != nil { - cb.logger.WithError(err).Error("[circuit breaker] failed to acquire lock") + isLeader = false + cb.logger.WithError(err).Debugf("[circuit breaker] failed to acquire lock") return err } defer func() { + sampleLatency := time.Since(start) + + // we are sleeping the rest of the duration because the sample might be done complete, + // but we don't want to release the lock until the next sample time window. + sleepTime := stopTime.Sub(time.Now()) + if sleepTime.Seconds() > 1.0 { + time.Sleep(sleepTime) + } + innerErr := cb.store.Unlock(ctx, mu) if innerErr != nil { - cb.logger.WithError(innerErr).Error("[circuit breaker] failed to unlock mutex") + cb.logger.WithError(innerErr).Debugf("[circuit breaker] failed to unlock mutex") + } + + if isLeader { + // should only be logged by the instance that runs the sample + cb.logger.Infof("[circuit breaker] sample completed in %v", sampleLatency) + + // cachedMetrics will be nil if metrics is not enabled + if cachedMetrics != nil { + // Update the sample latency metric + cachedMetrics.SampleLatency = sampleLatency + } } }() @@ -460,7 +484,7 @@ func (cb *CircuitBreakerManager) Start(ctx context.Context, pollFunc PollFunc) { return case <-ticker.C: if err := cb.sampleAndUpdate(ctx, pollFunc); err != nil { - cb.logger.WithError(err).Error("[circuit breaker] failed to sample and update circuit breakers") + cb.logger.WithError(err).Debug("[circuit breaker] failed to sample and update circuit breakers") } } } diff --git a/pkg/circuit_breaker/store.go b/pkg/circuit_breaker/store.go index 10fb37a8b9..a7b7a00db7 100644 --- a/pkg/circuit_breaker/store.go +++ b/pkg/circuit_breaker/store.go @@ -14,7 +14,7 @@ import ( ) type CircuitBreakerStore interface { - Lock(ctx context.Context, lockKey string) (*redsync.Mutex, error) + Lock(ctx context.Context, lockKey string, expiry uint64) (*redsync.Mutex, error) Unlock(ctx context.Context, mutex *redsync.Mutex) error Keys(context.Context, string) ([]string, error) GetOne(context.Context, string) (string, error) @@ -35,14 +35,14 @@ func NewRedisStore(redis redis.UniversalClient, clock clock.Clock) *RedisStore { } } -func (s *RedisStore) Lock(ctx context.Context, mutexKey string) (*redsync.Mutex, error) { +func (s *RedisStore) Lock(ctx context.Context, mutexKey string, expiry uint64) (*redsync.Mutex, error) { pool := goredis.NewPool(s.redis) rs := redsync.New(pool) ctx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() - mutex := rs.NewMutex(mutexKey, redsync.WithExpiry(time.Second), redsync.WithTries(1)) + mutex := rs.NewMutex(mutexKey, redsync.WithExpiry(time.Duration(expiry)*time.Second), redsync.WithTries(1)) err := mutex.LockContext(ctx) if err != nil { return nil, fmt.Errorf("failed to obtain lock: %v", err) @@ -57,7 +57,7 @@ func (s *RedisStore) Unlock(ctx context.Context, mutex *redsync.Mutex) error { ok, err := mutex.UnlockContext(ctx) if !ok { - return errors.New("failed to release lock") + return fmt.Errorf("failed to release lock: %v", err) } if err != nil { @@ -130,7 +130,7 @@ func NewTestStore() *TestStore { } } -func (t *TestStore) Lock(_ context.Context, _ string) (*redsync.Mutex, error) { +func (t *TestStore) Lock(_ context.Context, _ string, _ uint64) (*redsync.Mutex, error) { return nil, nil } diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index f42893ba2e..fd1ed4bac7 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -213,7 +213,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive // register latency mm := metrics.GetDPInstance(licenser) - mm.RecordLatency(eventDelivery) + mm.RecordEndToEndLatency(eventDelivery) } else { requestLogger.Errorf("%s", eventDelivery.UID) From 6052ee8aad452f099b348f5b97a7ed455662e7db Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 9 Oct 2024 16:56:34 +0200 Subject: [PATCH 47/48] feat: lint fixes --- pkg/circuit_breaker/circuit_breaker_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 6e86b8546f..4cdec07357 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -417,7 +417,7 @@ func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc P // we are sleeping the rest of the duration because the sample might be done complete, // but we don't want to release the lock until the next sample time window. - sleepTime := stopTime.Sub(time.Now()) + sleepTime := time.Until(stopTime) if sleepTime.Seconds() > 1.0 { time.Sleep(sleepTime) } From 1ebbaee23c615a0c21cd9b8117e29cf3dda3067c Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Wed, 9 Oct 2024 18:02:56 +0200 Subject: [PATCH 48/48] feat: add time at the end of the sample window --- pkg/circuit_breaker/circuit_breaker_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/circuit_breaker/circuit_breaker_manager.go b/pkg/circuit_breaker/circuit_breaker_manager.go index 4cdec07357..2716177f42 100644 --- a/pkg/circuit_breaker/circuit_breaker_manager.go +++ b/pkg/circuit_breaker/circuit_breaker_manager.go @@ -402,7 +402,7 @@ func (cb *CircuitBreakerManager) GetCircuitBreakerWithError(ctx context.Context, func (cb *CircuitBreakerManager) sampleAndUpdate(ctx context.Context, pollFunc PollFunc) error { start := time.Now() - stopTime := time.Now().Add(time.Duration(cb.config.SampleRate) * time.Second) + stopTime := time.Now().Add(time.Duration(cb.config.SampleRate-2) * time.Second) isLeader := true mu, err := cb.store.Lock(ctx, mutexKey, cb.config.SampleRate)