Skip to content

Commit

Permalink
materialize: log progress of requesting Open
Browse files Browse the repository at this point in the history
Getting the Open response for a materialization can take a long time, especially
if it has to do a lot of work to setup the transactor or if querying a
checkpoint from the store is not fast. This adds some progress logging for that
case so it's more clear what the materialization is doing.

Also a final round of refactoring: Split out the stream/logging logic into
separate files, and added a general `repeatAsync` helper that several of these
new logs make use of.
  • Loading branch information
williamhbaker committed Oct 4, 2024
1 parent 34e777a commit 4a05ddc
Show file tree
Hide file tree
Showing 3 changed files with 509 additions and 500 deletions.
13 changes: 10 additions & 3 deletions materialize-boilerplate/boilerplate.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"os"
"os/signal"
"syscall"
"time"

"net/http"
_ "net/http/pprof"
Expand Down Expand Up @@ -152,18 +153,24 @@ func materialize(ctx context.Context, stream m.MaterializeStream, connector Conn
return err
}
case request.Open != nil:
bl := newBindingEvents()
be := newBindingEvents()

transactor, opened, options, err := connector.NewTransactor(ctx, *request.Open, bl)
openStart := time.Now()
log.Info("requesting materialization Open")
stop := repeatAsync(func() { log.Info("materialization Open in progress") }, loggingFrequency)
transactor, opened, options, err := connector.NewTransactor(ctx, *request.Open, be)
if err != nil {
return err
}
stop(func() {
log.WithFields(log.Fields{"took": time.Since(openStart).String()}).Info("finished waiting for materialization Open")
})

if options == nil {
options = &MaterializeOptions{}
}

ts, err := newTransactionsStream(ctx, stream, lvl, *options, bl)
ts, err := newTransactionsStream(ctx, stream, lvl, *options, be)
if err != nil {
return fmt.Errorf("creating transactions stream: %w", err)
}
Expand Down
Loading

0 comments on commit 4a05ddc

Please sign in to comment.