Skip to content

Commit

Permalink
feat: added a consecutive error circuit breaker
Browse files Browse the repository at this point in the history
  • Loading branch information
Raj Nandan Sharma authored and Raj Nandan Sharma committed Apr 21, 2024
1 parent 5d1c4c8 commit 5da11f6
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 8 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
test:
go test -race
go test -coverprofile=coverage.out
go tool cover -html=coverage.out -o coverage.html
26 changes: 24 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,28 @@ if err != nil {
circuitOptions := tripper.CircuitOptions{
Name: "example-circuit",
Threshold: 10,
ThresholdType: tripper.ThresholdCount,
ThresholdType: tripper.ThresholdPercentage,
MinimumCount: 100,
IntervalInSeconds: 60,
OnCircuitOpen: onCircuitOpenCallback,
OnCircuitClosed: onCircuitClosedCallback,
}

circuit, err := tripper.ConfigureCircuit(circuitOptions)
if err != nil {
fmt.Println("Failed to add circuit:", err)
return
}
```

#### Circuit With Consecutive Errors
```go
//Adding a circuit that will trip the circuit if 10 consecutive erros occur in 1 minute
//for a minimum of 100 count
circuitOptions := tripper.CircuitOptions{
Name: "example-circuit",
Threshold: 10,
ThresholdType: tripper.ThresholdConsecutive,
MinimumCount: 100,
IntervalInSeconds: 60,
OnCircuitOpen: onCircuitOpenCallback,
Expand All @@ -69,6 +90,7 @@ if err != nil {
return
}
```

#### Circuit with Callbacks
```go
func onCircuitOpenCallback(x tripper.CallbackEvent){
Expand Down Expand Up @@ -105,7 +127,7 @@ if err != nil {
|---------------------|--------------------------------------------------------------|----------|------------|
| `Name` | The name of the circuit. | Required | `string` |
| `Threshold` | The threshold value for the circuit. | Required | `float32` |
| `ThresholdType` | The type of threshold (`ThresholdCount` or `ThresholdPercentage`). | Required | `string` |
| `ThresholdType` | The type of threshold (`ThresholdCount` or `ThresholdPercentage`or `ThresholdConsecutive`). | Required | `string` |
| `MinimumCount` | The minimum number of events required for monitoring. | Required | `int64` |
| `IntervalInSeconds` | The time interval for monitoring in seconds. | Required | `int` |
| `OnCircuitOpen` | Callback function called when the circuit opens. | Optional | `func()` |
Expand Down
25 changes: 20 additions & 5 deletions tripper.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ import (
// threshold type can be only COUNT or PERCENTAGE
// ThresholdCount represents a threshold type based on count.
const (
ThresholdCount = "COUNT"
ThresholdPercentage = "PERCENTAGE"
ThresholdCount = "COUNT"
ThresholdPercentage = "PERCENTAGE"
ThresholdConsecutive = "CONSECUTIVE"
)

var thresholdTypes = []string{ThresholdCount, ThresholdPercentage}
var thresholdTypes = []string{ThresholdCount, ThresholdPercentage, ThresholdConsecutive}

// Circuit represents a monitoring entity that tracks the status of a circuit.
type Circuit interface {
Expand Down Expand Up @@ -47,6 +48,7 @@ type CircuitImplementation struct {
CircuitOpen bool // Indicates whether the circuit is open or closed
LastCapturedAt int64 // Timestamp of the last captured event
CircuitOpenedSince int64 // Timestamp when the circuit was opened
ConsecutiveCounter int64
Ticker *time.Ticker
Mutex sync.Mutex
XMutex sync.Mutex
Expand Down Expand Up @@ -105,7 +107,8 @@ func ConfigureCircuit(monitorOptions CircuitOptions) (Circuit, error) {
}

newMonitor := &CircuitImplementation{
Options: monitorOptions,
Options: monitorOptions,
ConsecutiveCounter: 0,
}
newMonitor.Ticker = time.NewTicker(time.Duration(monitorOptions.IntervalInSeconds) * time.Second)
go func() {
Expand All @@ -115,6 +118,7 @@ func ConfigureCircuit(monitorOptions CircuitOptions) (Circuit, error) {
newMonitor.SuccessCount = 0
newMonitor.FailureCount = 0
newMonitor.CircuitOpenedSince = 0
newMonitor.ConsecutiveCounter = 0
newMonitor.CircuitOpen = false
if newMonitor.Options.OnCircuitClosed != nil {
newMonitor.Options.OnCircuitClosed(CallbackEvent{
Expand All @@ -137,8 +141,10 @@ func (m *CircuitImplementation) UpdateStatus(success bool) {

m.LastCapturedAt = getTimestamp()
if success {
m.ConsecutiveCounter = 0
m.SuccessCount++
} else {
m.ConsecutiveCounter++
m.FailureCount++
}
if m.SuccessCount+m.FailureCount < m.Options.MinimumCount {
Expand All @@ -155,7 +161,7 @@ func (m *CircuitImplementation) UpdateStatus(success bool) {
m.CircuitOpen = false
m.CircuitOpenedSince = 0
}
} else {
} else if m.Options.ThresholdType == ThresholdPercentage {
// if the threshold type is percentage, check if the percentage of failures is greater than the threshold
totalRequests := m.FailureCount + m.SuccessCount
failurePercentage := (m.FailureCount * 100) / totalRequests
Expand All @@ -167,6 +173,15 @@ func (m *CircuitImplementation) UpdateStatus(success bool) {
m.CircuitOpenedSince = 0

}
} else if m.Options.ThresholdType == ThresholdConsecutive {
if m.ConsecutiveCounter >= int64(m.Options.Threshold) {
m.CircuitOpen = true
m.CircuitOpenedSince = m.LastCapturedAt
} else {
m.CircuitOpen = false
m.CircuitOpenedSince = 0
}

}
if currentStateOfCircuit != m.CircuitOpen && m.CircuitOpen {

Expand Down
30 changes: 29 additions & 1 deletion tripper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func TestUpdateStatus(t *testing.T) {
assert.False(t, m.Data().IsCircuitOpen)

// Test case 4: Update status with success=false and minimum count not reached
// Expected output: Failure count incremented, circuit not opened
// Expected output: Failure count incremented, circuit opened
m.UpdateStatus(false)
assert.Equal(t, int64(2), m.Data().SuccessCount)
assert.Equal(t, int64(2), m.Data().FailureCount)
Expand Down Expand Up @@ -365,3 +365,31 @@ func TestIsCircuitOpen(t *testing.T) {
m.UpdateStatus(true)
assert.False(t, m.IsCircuitOpen())
}
func TestUpdateStatusConsecutive(t *testing.T) {
monitorOptions := CircuitOptions{
Name: "TEST_ThresholdConsecutive",
Threshold: 5,
MinimumCount: 2,
IntervalInSeconds: 120,
ThresholdType: ThresholdConsecutive,
}
m, err := ConfigureCircuit(monitorOptions)
assert.NoError(t, err)
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(false)
assert.True(t, m.IsCircuitOpen())
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(true)
assert.False(t, m.IsCircuitOpen())
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(false)
m.UpdateStatus(false)
assert.True(t, m.IsCircuitOpen())

}

0 comments on commit 5da11f6

Please sign in to comment.