Skip to content

Commit

Permalink
[patch] exported trigger type to be able to configure processor from …
Browse files Browse the repository at this point in the history
…external packages

[-] updated README with example
  • Loading branch information
bnkamalesh committed Nov 5, 2024
1 parent a33d970 commit 585a2d7
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 14 deletions.
106 changes: 103 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ The same concept can also be extended to handle events processing. So, we have a

The processing of a single micro batch can be triggered in two ways, based on a time ticker or if the micro batch size is full. i.e. process a non empty batch if duration X has passed or if the batch size is full

## How to use nibbler?

### Config

```golang
Expand All @@ -44,7 +42,109 @@ type Config[T any] struct {
}
```

You can find usage details in the tests.
## How to use nibbler?

```golang
package main

import (
"context"
"fmt"
"sync"
"time"

"github.com/naughtygopher/nibbler"
)

type db struct {
data sync.Map
totalBalance int
}

func (d *db) BulkAddAccountsAndBalance(pp []AccStatement) error {
// assume we are doing a bulk insert/update into the database instead of inserting one by one.
// Bulk operations reduce the number of I/O required between your application and the database.
// Thereby making it better in most cases.
for _, p := range pp {
d.data.Store(p.AccountID, p.Balance)
d.totalBalance += p.Balance
}
return nil
}

type Bank struct {
db *db
}

func (bnk *Bank) ProcessAccountsBatch(
ctx context.Context,
trigger nibbler.Trigger,
batch []AccStatement,
) error {
err := bnk.db.BulkAddAccountsAndBalance(batch)
if err != nil {
return err
}

return nil
}

func (bnk *Bank) TotalBalance() int {
return bnk.db.totalBalance
}

func (bnk *Bank) TotalAccounts() int {
counter := 0
bnk.db.data.Range(func(key, value any) bool {
counter++
return true
})
return counter
}

type AccStatement struct {
AccountID string
Balance int
}

func main() {
bnk := Bank{
db: &db{
data: sync.Map{},
},
}

nib, err := nibbler.Start(&nibbler.Config[AccStatement]{
Size: 10,
TickerDuration: time.Second,
Processor: bnk.ProcessAccountsBatch,
})
if err != nil {
panic(err)
}

receiver := nib.Receiver()
for i := range 100 {
accID := fmt.Sprintf("account_id_%d", i)
receiver <- AccStatement{
AccountID: accID,
Balance: 50000 / (i + 1),
}
}

// wait for batches to be processed. Ideally this wouldn't be required as our application
// would not exit, instead just keep listening to the events stream.
time.Sleep(time.Second)

fmt.Printf(
"Number of accounts %d, total balance: %d\n",
bnk.TotalAccounts(),
bnk.TotalBalance(),
)
}
```

You can find all usage details in the tests.

## The gopher

Expand Down
10 changes: 5 additions & 5 deletions nibbler.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ import (

var ErrValidation = errors.New("validation failed")

type trigger string
type Trigger string

const (
TriggerTicker trigger = "TICKER"
TriggerFull trigger = "BATCH_FULL"
TriggerTicker Trigger = "TICKER"
TriggerFull Trigger = "BATCH_FULL"
)

type BatchProcessor[T any] func(ctx context.Context, trigger trigger, batch []T) error
type BatchProcessor[T any] func(ctx context.Context, trigger Trigger, batch []T) error

type Config[T any] struct {
// ProcessingTimeout is context timeout for processing a single batch
Expand Down Expand Up @@ -88,7 +88,7 @@ func (bat *Nibbler[T]) panicRecovery(rec any, err error) error {
return err
}

func (bat *Nibbler[T]) processBatch(trigger trigger) (err error) {
func (bat *Nibbler[T]) processBatch(trigger Trigger) (err error) {
defer func() {
err = bat.panicRecovery(recover(), err)
}()
Expand Down
12 changes: 6 additions & 6 deletions nibbler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestNibbler(tt *testing.T) {
Size: batchSize,
ProcessingTimeout: time.Millisecond,
TickerDuration: batchFreq,
Processor: func(_ context.Context, _ trigger, batch []string) error {
Processor: func(_ context.Context, _ Trigger, batch []string) error {
gotchan <- slices.Clone(batch)

// last val is used to identify the end of all batches so that
Expand Down Expand Up @@ -99,7 +99,7 @@ func TestProcessorErr(tt *testing.T) {

nib, err := Start(&Config[string]{
TickerDuration: time.Second,
Processor: func(_ context.Context, _ trigger, _ []string) error {
Processor: func(_ context.Context, _ Trigger, _ []string) error {
return errProcessing
},
ProcessorErr: func(failedBatch []string, err error) {
Expand Down Expand Up @@ -128,7 +128,7 @@ func TestProcessorErr(tt *testing.T) {
assertElement := "hello"
nib, err := Start(&Config[string]{
TickerDuration: time.Second,
Processor: func(_ context.Context, _ trigger, _ []string) error {
Processor: func(_ context.Context, _ Trigger, _ []string) error {
return errProcessing
},
ProcessorErr: func(failedBatch []string, err error) {
Expand Down Expand Up @@ -160,7 +160,7 @@ func TestProcessorErr(tt *testing.T) {

nib, err := Start(&Config[string]{
TickerDuration: time.Second,
Processor: func(_ context.Context, _ trigger, _ []string) error {
Processor: func(_ context.Context, _ Trigger, _ []string) error {
panic(errProcessing)
},
ProcessorErr: func(failedBatch []string, err error) {
Expand Down Expand Up @@ -192,7 +192,7 @@ func TestProcessorErr(tt *testing.T) {
assertElement := "hello"
nib, err := Start(&Config[string]{
TickerDuration: time.Second,
Processor: func(_ context.Context, _ trigger, _ []string) error {
Processor: func(_ context.Context, _ Trigger, _ []string) error {
panic("processor panic")
},
ProcessorErr: func(failedBatch []string, err error) {
Expand All @@ -214,7 +214,7 @@ func TestProcessorErr(tt *testing.T) {

func TestSanitizeValidate(tt *testing.T) {
var (
processor = func(ctx context.Context, trigger trigger, batch []string) error { return nil }
processor = func(ctx context.Context, trigger Trigger, batch []string) error { return nil }
)

tt.Run("all valid", func(t *testing.T) {
Expand Down

0 comments on commit 585a2d7

Please sign in to comment.