From e0d0ed9735f114a711bf859eda02eb41687cdfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 27 Sep 2024 09:28:58 +0200 Subject: [PATCH 1/5] fix: There should be flush lock to prevent incorrect write into gzip file --- .../stream/storage/level/local/encoding/pipeline.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go b/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go index 72aa0a29a4..248c5b9989 100644 --- a/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go +++ b/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go @@ -78,9 +78,10 @@ type NetworkOutput interface { // pipeline implements Pipeline interface, it wraps common logic for all file types. // For conversion between record values and bytes, the encoder.Encoder is used. type pipeline struct { - logger log.Logger - sliceKey model.SliceKey - events *events.Events[Pipeline] + logger log.Logger + sliceKey model.SliceKey + events *events.Events[Pipeline] + flushLock sync.RWMutex encoder encoder.Encoder chain *writechain.Chain @@ -270,6 +271,8 @@ func (p *pipeline) WriteRecord(record recordctx.Context) (int, error) { return 0, errors.New(`writer is closed`) } + p.flushLock.RLock() + defer p.flushLock.RUnlock() // Format and write table row n, err := p.encoder.WriteRecord(record) p.writeWg.Done() @@ -333,6 +336,9 @@ func (p *pipeline) UncompressedSize() datasize.ByteSize { // Flush all internal buffers to the NetworkOutput. // The method is called periodically by the writesync.Syncer. func (p *pipeline) Flush(ctx context.Context) error { + p.flushLock.Lock() + defer p.flushLock.Unlock() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() From f0bb6bd8c99024d7d6cb94e17970ad84dfc34987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 27 Sep 2024 10:20:09 +0200 Subject: [PATCH 2/5] fix: Add dbCtx to incrementRetryAttempt not work with canceled context Add new log message for statistics mod revision --- .../node/coordinator/filerotation/operator.go | 14 +++++++++----- .../node/coordinator/slicerotation/operator.go | 15 +++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go b/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go index 1164dc9822..74da734eeb 100644 --- a/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go +++ b/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go @@ -369,13 +369,17 @@ func (o *operator) rotateFile(ctx context.Context, file *fileData) { err = o.storage.File().Rotate(file.FileKey.SinkKey, o.clock.Now()).RequireLock(lock).Do(ctx).Err() // Handle error if err != nil { - rb.InvokeIfErr(ctx, &err) - o.logger.Errorf(ctx, "cannot rotate file: %s", err) + // Update the entity, the ctx may be cancelled + dbCtx, dbCancel := context.WithTimeout(context.WithoutCancel(ctx), dbOperationTimeout) + defer dbCancel() + + rb.InvokeIfErr(dbCtx, &err) + o.logger.Errorf(dbCtx, "cannot rotate file: %s", err) // Increment retry delay - rErr := o.storage.File().IncrementRetryAttempt(file.FileKey, o.clock.Now(), err.Error()).RequireLock(lock).Do(ctx).Err() + rErr := o.storage.File().IncrementRetryAttempt(file.FileKey, o.clock.Now(), err.Error()).RequireLock(lock).Do(dbCtx).Err() if rErr != nil { - o.logger.Errorf(ctx, "cannot increment file rotation retry attempt: %s", rErr) + o.logger.Errorf(dbCtx, "cannot increment file rotation retry attempt: %s", rErr) return } } @@ -441,7 +445,7 @@ func (o *operator) closeFile(ctx context.Context, file *fileData) { // If there is an error, increment retry delay if err != nil { o.logger.Error(dbCtx, err.Error()) - fileEntity, rErr := o.storage.File().IncrementRetryAttempt(file.FileKey, o.clock.Now(), err.Error()).RequireLock(lock).Do(ctx).ResultOrErr() + fileEntity, rErr := o.storage.File().IncrementRetryAttempt(file.FileKey, o.clock.Now(), err.Error()).RequireLock(lock).Do(dbCtx).ResultOrErr() if rErr != nil { o.logger.Errorf(ctx, "cannot increment file close retry: %s", rErr) return diff --git a/internal/pkg/service/stream/storage/node/coordinator/slicerotation/operator.go b/internal/pkg/service/stream/storage/node/coordinator/slicerotation/operator.go index e36e9883d0..8cc0b3ecf1 100644 --- a/internal/pkg/service/stream/storage/node/coordinator/slicerotation/operator.go +++ b/internal/pkg/service/stream/storage/node/coordinator/slicerotation/operator.go @@ -297,15 +297,18 @@ func (o *operator) rotateSlice(ctx context.Context, slice *sliceData) { // Handle error if err != nil { var stateErr sliceRepo.UnexpectedFileSliceStatesError + // Update the entity, the ctx may be cancelled + dbCtx, dbCancel := context.WithTimeout(context.WithoutCancel(ctx), dbOperationTimeout) + defer dbCancel() if errors.As(err, &stateErr) && stateErr.FileState != model.FileWriting { - o.logger.Info(ctx, "skipped slice rotation, file is already closed") + o.logger.Info(dbCtx, "skipped slice rotation, file is already closed") } else { - o.logger.Errorf(ctx, "cannot rotate slice: %s", err) + o.logger.Errorf(dbCtx, "cannot rotate slice: %s", err) // Increment retry delay - rErr := o.storage.Slice().IncrementRetryAttempt(slice.SliceKey, o.clock.Now(), err.Error()).RequireLock(lock).Do(ctx).Err() + rErr := o.storage.Slice().IncrementRetryAttempt(slice.SliceKey, o.clock.Now(), err.Error()).RequireLock(lock).Do(dbCtx).Err() if rErr != nil { - o.logger.Errorf(ctx, "cannot increment file rotation retry attempt: %s", err) + o.logger.Errorf(dbCtx, "cannot increment file rotation retry attempt: %s", err) return } } @@ -364,7 +367,7 @@ func (o *operator) closeSlice(ctx context.Context, slice *sliceData) { // If there is an error, increment retry delay if err != nil { o.logger.Error(dbCtx, err.Error()) - sliceEntity, rErr := o.storage.Slice().IncrementRetryAttempt(slice.SliceKey, o.clock.Now(), err.Error()).Do(ctx).ResultOrErr() + sliceEntity, rErr := o.storage.Slice().IncrementRetryAttempt(slice.SliceKey, o.clock.Now(), err.Error()).Do(dbCtx).ResultOrErr() if rErr != nil { o.logger.Errorf(ctx, "cannot increment slice retry: %s", rErr) return @@ -393,7 +396,7 @@ func (o *operator) waitForSliceClosing(ctx context.Context, slice *sliceData) (s // Make sure the statistics cache is up-to-date if err := o.statisticsCache.WaitForRevision(ctx, slice.ModRevision); err != nil { - return statistics.Aggregated{}, errors.PrefixError(err, "error when waiting for statistics cache revision") + return statistics.Aggregated{}, errors.PrefixErrorf(err, "error when waiting for statistics cache revision, actual: %v, expected: %v", o.statisticsCache.Revision(), slice.ModRevision) } // Get slice statistics From 7284e30e8140ad817b7e8afc70104b2a57a667ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Jure=C4=8Dko?= Date: Fri, 27 Sep 2024 10:35:56 +0200 Subject: [PATCH 3/5] tests: Disable unrelated workers --- test/stream/bridge/keboola/keboola_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/stream/bridge/keboola/keboola_test.go b/test/stream/bridge/keboola/keboola_test.go index 395e497471..8f9fb01bc3 100644 --- a/test/stream/bridge/keboola/keboola_test.go +++ b/test/stream/bridge/keboola/keboola_test.go @@ -34,6 +34,11 @@ func TestKeboolaBridgeWorkflow(t *testing.T) { // Update configuration to make the cluster testable configFn := func(cfg *config.Config) { + // Disable unrelated workers + cfg.Storage.DiskCleanup.Enabled = false + cfg.Storage.MetadataCleanup.Enabled = false + cfg.API.Task.CleanupEnabled = false + // Use deterministic load balancer cfg.Storage.Level.Local.Writer.Network.PipelineBalancer = network.RoundRobinBalancerType From 22b023b958c97f9fbe97b23ef8c20a0778105acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Jure=C4=8Dko?= Date: Fri, 27 Sep 2024 11:00:54 +0200 Subject: [PATCH 4/5] feat: Add align warning --- .../storage/level/local/diskwriter/writer.go | 1 + .../storage/level/local/encoding/pipeline.go | 6 +- out.txt | 1469 +++++++++++++++++ 3 files changed, 1473 insertions(+), 3 deletions(-) create mode 100644 out.txt diff --git a/internal/pkg/service/stream/storage/level/local/diskwriter/writer.go b/internal/pkg/service/stream/storage/level/local/diskwriter/writer.go index 2930fcddcd..d408656e30 100644 --- a/internal/pkg/service/stream/storage/level/local/diskwriter/writer.go +++ b/internal/pkg/service/stream/storage/level/local/diskwriter/writer.go @@ -170,6 +170,7 @@ func (w *writer) Close(ctx context.Context) error { w.wg.Wait() if w.writen != w.aligned { + w.logger.Warnf(ctx, `file is not aligned, truncating`) seeked, err := w.file.Seek(w.aligned-w.writen, io.SeekCurrent) if err == nil { err = w.file.Truncate(seeked) diff --git a/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go b/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go index 248c5b9989..14916c02d9 100644 --- a/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go +++ b/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go @@ -271,18 +271,18 @@ func (p *pipeline) WriteRecord(record recordctx.Context) (int, error) { return 0, errors.New(`writer is closed`) } - p.flushLock.RLock() - defer p.flushLock.RUnlock() // Format and write table row + p.flushLock.RLock() n, err := p.encoder.WriteRecord(record) p.writeWg.Done() + p.flushLock.RUnlock() if err != nil { return n, err } notifier := p.syncer.Notifier() - // Increments number of high-level writes in progress + // Increments number of high-level writes in progressd p.acceptedWrites.Add(timestamp, 1) // Wait for sync and return sync error, if any diff --git a/out.txt b/out.txt new file mode 100644 index 0000000000..844ca48e35 --- /dev/null +++ b/out.txt @@ -0,0 +1,1469 @@ +? github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/encoder/csv/fastcsv [no test files] +=== RUN TestCSVWriter +=== PAUSE TestCSVWriter +=== RUN TestCSVWriterAboveLimit +=== PAUSE TestCSVWriterAboveLimit +=== CONT TestCSVWriter +=== CONT TestCSVWriterAboveLimit +=== RUN TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false +=== PAUSE TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false +--- PASS: TestCSVWriterAboveLimit (0.00s) +=== CONT TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false +{"level":"info","time":"2024-09-27T08:53:29.429Z","message":"connecting to etcd","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} +{"level":"info","time":"2024-09-27T08:53:29.490Z","message":"connected to etcd cluster \"etcd:2379\"","duration":"61.351095ms","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} +{"level":"info","time":"2024-09-27T08:53:29.493Z","message":"creating etcd session","component":"distribution.mutex.provider.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.556Z","message":"created etcd session","duration":"63.125746ms","component":"distribution.mutex.provider.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.616Z","message":"watch stream created","nodeId":"api","stream.prefix":"storage/volume/writer/","component":"volume.repository"} +{"level":"info","time":"2024-09-27T08:53:29.616Z","message":"node ID \"api\"","nodeId":"api","node":"api","component":"task"} +{"level":"info","time":"2024-09-27T08:53:29.616Z","message":"creating etcd session","nodeId":"api","node":"api","component":"task.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.678Z","message":"created etcd session","nodeId":"api","duration":"61.823622ms","node":"api","component":"task.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.679Z","message":"connecting to etcd","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} +{"level":"info","time":"2024-09-27T08:53:29.737Z","message":"connected to etcd cluster \"etcd:2379\"","duration":"58.314469ms","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} +{"level":"info","time":"2024-09-27T08:53:29.738Z","message":"creating etcd session","component":"distribution.mutex.provider.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.798Z","message":"created etcd session","duration":"60.116063ms","component":"distribution.mutex.provider.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.857Z","message":"watch stream created","nodeId":"disk-writer","stream.prefix":"storage/volume/writer/","component":"volume.repository"} +{"level":"info","time":"2024-09-27T08:53:29.857Z","message":"searching for volumes in volumes path","nodeId":"disk-writer","volumes.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002","component":"storage.node.writer.volumes"} +{"level":"info","time":"2024-09-27T08:53:29.857Z","message":"found volume","nodeId":"disk-writer","volume.label":"001","volume.type":"hdd","component":"storage.node.writer.volumes"} +{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"opening volume","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","component":"storage.node.writer.volumes.volume"} +{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"opened volume","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.type":"hdd","volume.label":"001","component":"storage.node.writer.volumes.volume"} +{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"found \"1\" volumes","nodeId":"disk-writer","component":"storage.node.writer.volumes"} +{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"starting storage writer node","nodeId":"disk-writer","component":"storage.node.writer"} +{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"disk writer listening on \"[::]:33631\"","nodeId":"disk-writer","listenAddress":"0.0.0.0:33631","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} +{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"creating etcd session","nodeId":"disk-writer","component":"volumes.registry.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:29.925Z","message":"created etcd session","nodeId":"disk-writer","duration":"66.514211ms","component":"volumes.registry.etcd.session"} +{"level":"debug","time":"2024-09-27T08:53:29.926Z","message":"watch stream mirror synced to revision 1357","nodeId":"api","stream.prefix":"storage/volume/writer/","component":"volume.repository"} +{"level":"debug","time":"2024-09-27T08:53:29.926Z","message":"watch stream mirror synced to revision 1357","nodeId":"disk-writer","stream.prefix":"storage/volume/writer/","component":"volume.repository"} +{"level":"info","time":"2024-09-27T08:53:29.927Z","message":"registered \"1\" volumes","nodeId":"disk-writer","component":"volumes.registry"} +{"level":"info","time":"2024-09-27T08:53:29.927Z","message":"removing expired files without DB record from disk","nodeId":"disk-writer","component":"storage.disk.cleanup"} +{"level":"info","time":"2024-09-27T08:53:29.928Z","message":"connecting to etcd","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} +{"level":"info","time":"2024-09-27T08:53:29.928Z","message":"removed \"0\" directories","nodeId":"disk-writer","removedDirectoriesCount":0,"component":"storage.disk.cleanup"} +{"level":"info","time":"2024-09-27T08:53:29.988Z","message":"connected to etcd cluster \"etcd:2379\"","duration":"59.437793ms","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} +{"level":"info","time":"2024-09-27T08:53:29.989Z","message":"creating etcd session","component":"distribution.mutex.provider.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:30.049Z","message":"created etcd session","duration":"59.768429ms","component":"distribution.mutex.provider.etcd.session"} +{"level":"debug","time":"2024-09-27T08:53:30.049Z","message":"watch stream mirror synced to revision 1357","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"volume.repository"} +{"level":"info","time":"2024-09-27T08:53:30.107Z","message":"watch stream created","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"volume.repository"} +{"level":"debug","time":"2024-09-27T08:53:30.107Z","message":"watch stream mirror synced to revision 1357","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"storage.router.connections"} +{"level":"info","time":"2024-09-27T08:53:30.107Z","message":"the list of volumes has changed, updating connections","nodeId":"source","component":"storage.router.connections"} +{"level":"debug","time":"2024-09-27T08:53:30.107Z","message":"new disk writer node \"disk-writer\"","nodeId":"source","component":"storage.router.connections"} +{"level":"debug","time":"2024-09-27T08:53:30.107Z","message":"registered connection to \"disk-writer\", total connections count 1","nodeId":"source","nodeId":"source","remoteAddr":"localhost:33631","remoteNodeID":"disk-writer","component":"storage.router.connections.client.transport"} +{"level":"info","time":"2024-09-27T08:53:30.107Z","message":"disk writer client is connecting to \"disk-writer\" - \"localhost:33631\"","nodeId":"source","nodeId":"source","remoteAddr":"localhost:33631","remoteNodeID":"disk-writer","component":"storage.router.connections.client.transport"} +{"level":"info","time":"2024-09-27T08:53:30.108Z","message":"disk writer client connected from \"127.0.0.1:54770\" to \"disk-writer\" - \"localhost:33631\"","nodeId":"source","nodeId":"source","remoteAddr":"localhost:33631","remoteNodeID":"disk-writer","component":"storage.router.connections.client.transport"} +{"level":"info","time":"2024-09-27T08:53:30.108Z","message":"accepted connection from \"127.0.0.1:54770\" to \"127.0.0.1:33631\"","nodeId":"disk-writer","listenAddress":"0.0.0.0:33631","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} +{"level":"debug","time":"2024-09-27T08:53:30.108Z","message":"registered session to \"127.0.0.1:54770\", total sessions count 1","nodeId":"disk-writer","listenAddress":"0.0.0.0:33631","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} +{"level":"info","time":"2024-09-27T08:53:30.165Z","message":"watch stream created","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"storage.router.connections"} +{"level":"info","time":"2024-09-27T08:53:30.225Z","message":"watch stream created","nodeId":"source","stream.prefix":"definition/sink/active/","component":"sink.router"} +{"level":"info","time":"2024-09-27T08:53:30.225Z","message":"joining distribution group","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:30.225Z","message":"creating etcd session","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:30.284Z","message":"created etcd session","nodeId":"source","duration":"59.522845ms","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:30.284Z","message":"registering the node \"source\"","nodeId":"source","node":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:30.286Z","message":"the node \"source\" registered","nodeId":"source","duration":"1.177936ms","node":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:30.286Z","message":"watching for other nodes","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:30.286Z","message":"found a new node \"source\"","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:30.344Z","message":"watch stream created","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","stream.prefix":"","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:30.344Z","message":"creating etcd session","nodeId":"source","component":"close-sync.source.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:30.401Z","message":"created etcd session","nodeId":"source","duration":"57.140794ms","component":"close-sync.source.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:30.460Z","message":"watch stream created","nodeId":"source","stream.prefix":"storage/slice/level/local/","component":"storage.router"} +{"level":"debug","time":"2024-09-27T08:53:30.472Z","message":"executing OnBranchSave \"123/111\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.474Z","message":"executing OnSourceSave \"123/111/my-source\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.478Z","message":"executing OnSinkSave \"123/111/my-source/my-sink\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.479Z","message":"executing OnFileOpen \"123/111/my-source/my-sink\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.479Z","message":"executing OnFileSave \"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.480Z","message":"executing OnSliceOpen \"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.480Z","message":"executing OnSliceSave \"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z\"","nodeId":"api","component":"plugin"} +{"level":"debug","time":"2024-09-27T08:53:30.484Z","message":"watch stream mirror synced to revision 1362","nodeId":"source","component":"sink.router"} +{"level":"debug","time":"2024-09-27T08:53:30.484Z","message":"watch stream mirror synced to revision 1362","nodeId":"source","stream.prefix":"storage/slice/level/local/","component":"storage.router"} +{"level":"info","time":"2024-09-27T08:53:30.498Z","message":"opening sink pipeline","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","branch.id":"111","project.id":"123","sink.id":"my-sink","source.id":"my-source","component":"sink.router"} +{"level":"debug","time":"2024-09-27T08:53:30.499Z","message":"registered stream \"1\" to \"127.0.0.1:54770\", total streams count 1","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} +{"level":"debug","time":"2024-09-27T08:53:30.500Z","message":"opening disk writer","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","component":"storage.node.writer.volumes.volume"} +{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"opened file","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","file.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001/123/111/my-source/my-sink/2024-09-27T08-53-30-477Z/2024-09-27T08-53-30-477Z/slice-source.csv","component":"storage.node.writer.volumes.volume"} +{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"allocated disk space \"100MB\"","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","file.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001/123/111/my-source/my-sink/2024-09-27T08-53-30-477Z/2024-09-27T08-53-30-477Z/slice-source.csv","component":"storage.node.writer.volumes.volume"} +{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"opened disk writer","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","component":"storage.node.writer.volumes.volume"} +{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"opening encoding pipeline","nodeId":"source","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice":"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer","component":"encoding.pipeline"} +{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"sync is enabled, mode=cache, sync each {count=5000 or uncompressed=10MB or compressed=1MB or interval=10ms}, check each 5ms","nodeId":"source","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice":"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer"} +{"level":"debug","time":"2024-09-27T08:53:30.502Z","message":"opened encoding pipeline","nodeId":"source","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice":"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer","component":"encoding.pipeline"} +{"level":"info","time":"2024-09-27T08:53:30.502Z","message":"opened slice pipeline","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer","component":"storage.router"} +{"level":"info","time":"2024-09-27T08:53:30.502Z","message":"opened sink pipeline to 1 slices","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","branch.id":"111","project.id":"123","sink.id":"my-sink","source.id":"my-source","component":"storage.router"} +{"level":"info","time":"2024-09-27T08:53:30.502Z","message":"opened sink pipeline","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","branch.id":"111","project.id":"123","sink.id":"my-sink","source.id":"my-source","component":"sink.router"} +{"level":"debug","time":"2024-09-27T08:53:30.512Z","message":"starting sync to cache","nodeId":"source"} +{"level":"debug","time":"2024-09-27T08:53:31.182Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:32.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} + testcase.go:144: + Error Trace: /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:144 + Error: timeout when waiting for batch 102 + Test: TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false +{"level":"info","time":"2024-09-27T08:53:32.499Z","message":"exiting (test cleanup)"} +{"level":"info","time":"2024-09-27T08:53:32.499Z","message":"closing close-sync source node","nodeId":"source","component":"close-sync.source"} +{"level":"info","time":"2024-09-27T08:53:32.499Z","message":"closing etcd session: context canceled","nodeId":"source","component":"close-sync.source.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"closed etcd session","nodeId":"source","duration":"15.671847ms","component":"close-sync.source.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"closed close-sync source node","nodeId":"source","component":"close-sync.source"} +{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"received shutdown request","nodeId":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"unregistering the node \"source\"","nodeId":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"watch stream consumer closed: context canceled","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","stream.prefix":"","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:32.519Z","message":"the node \"source\" unregistered","nodeId":"source","duration":"4.544208ms","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:32.519Z","message":"closing etcd session: context canceled","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"closed etcd session","nodeId":"source","duration":"1.473996ms","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} +{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"shutdown done","nodeId":"source","component":"distribution"} +{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"closing storage router","nodeId":"source","component":"storage.router"} +{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"watch stream consumer closed: context canceled","nodeId":"source","stream.prefix":"storage/slice/level/local/","component":"storage.router"} +{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"closing 1 sink pipelines","nodeId":"source","component":"storage.router"} +{"level":"debug","time":"2024-09-27T08:53:32.521Z","message":"closing sink pipeline: shutdown","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","component":"storage.router"} +{"level":"debug","time":"2024-09-27T08:53:33.166Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:34.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:35.166Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:36.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:37.166Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:38.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +{"level":"debug","time":"2024-09-27T08:53:39.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} +panic: test timed out after 10s +running tests: + TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false (10s) + +goroutine 306 [running]: +testing.(*M).startAlarm.func1() + /usr/local/go/src/testing/testing.go:2366 +0x385 +created by time.goFunc + /usr/local/go/src/time/sleep.go:177 +0x2d + +goroutine 1 [chan receive]: +testing.tRunner.func1() + /usr/local/go/src/testing/testing.go:1650 +0x4ab +testing.tRunner(0xc000aa69c0, 0xc0009bfc70) + /usr/local/go/src/testing/testing.go:1695 +0x134 +testing.runTests(0xc000aa8828, {0x3e073c0, 0x2, 0x2}, {0x1?, 0x51fa6e?, 0x4089a40?}) + /usr/local/go/src/testing/testing.go:2159 +0x445 +testing.(*M).Run(0xc000a88e60) + /usr/local/go/src/testing/testing.go:2027 +0x68b +main.main() + _testmain.go:53 +0x16c + +goroutine 11 [select]: +go.opencensus.io/stats/view.(*worker).start(0xc0005cad00) + /tmp/cache/go-mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x9f +created by go.opencensus.io/stats/view.init.0 in goroutine 1 + /tmp/cache/go-mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x8d + +goroutine 66 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() + /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 + +goroutine 13 [chan receive]: +testing.tRunner.func1() + /usr/local/go/src/testing/testing.go:1650 +0x4ab +testing.tRunner(0xc000aa6b60, 0x24cc220) + /usr/local/go/src/testing/testing.go:1695 +0x134 +created by testing.(*T).Run in goroutine 1 + /usr/local/go/src/testing/testing.go:1742 +0x390 + +goroutine 58 [select]: +go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000141ae0, {0x2b39318, 0xc0008be220}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:590 +0x1d2 +created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 25 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:500 +0x2ab + +goroutine 15 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.(*Process).WaitForShutdown(...) + /code/internal/pkg/service/common/servicectx/servicectx.go:194 +github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.NewForTest.func1() + /code/internal/pkg/service/common/servicectx/servicectx.go:143 +0xe7 +testing.(*common).Cleanup.func1() + /usr/local/go/src/testing/testing.go:1175 +0x10f +testing.(*common).runCleanup(0xc000aa6ea0, 0x0?) + /usr/local/go/src/testing/testing.go:1353 +0xdb +testing.tRunner.func2() + /usr/local/go/src/testing/testing.go:1683 +0x25 +runtime.Goexit() + /usr/local/go/src/runtime/panic.go:626 +0x5e +testing.(*common).FailNow(0xc000aa6ea0) + /usr/local/go/src/testing/testing.go:1005 +0x4a +github.com/stretchr/testify/require.Fail({0x2b19b88, 0xc000aa6ea0}, {0xc00106e150, 0x22}, {0x0, 0x0, 0x0}) + /tmp/cache/go-mod/github.com/stretchr/testify@v1.9.0/require/require.go:508 +0xca +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testcase.(*WriterTestCase).Run(0xc000a8b180, 0xc000aa6ea0) + /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:144 +0x126b +testing.tRunner(0xc000aa6ea0, 0xc00088f400) + /usr/local/go/src/testing/testing.go:1689 +0xfb +created by testing.(*T).Run in goroutine 13 + /usr/local/go/src/testing/testing.go:1742 +0x390 + +goroutine 34 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New.func1() + /code/internal/pkg/service/common/servicectx/servicectx.go:100 +0x26 +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New in goroutine 15 + /code/internal/pkg/service/common/servicectx/servicectx.go:98 +0x23c + +goroutine 43 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b0f470, {0x2b2cc60, 0xc000a87cc0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 57 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 25 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 25 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00013f2c0, {0xc0008bafd0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00013f2c0, {0xc0008bafd0?, 0xc000726e88?, 0xc000c059b8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0008baf60, {0xc0008bafd0?, 0xc000c05a30?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc0008baf60}, {0xc0008bafd0, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008bc5a0, {0xc0008bafd0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc0008bafc0, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc0008bafc0, 0xc0008bc5a0, {0x0, 0x0}, 0x7fffffff, 0xc000c05d38, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0xc000c05c98?, {0x7f992fe94558, 0x40f8680}, 0xc000c05ca0?, {0x0?, 0x0?}, {0x2340c20, 0xc000716e80}, 0xc000c05cb8?, 0xc000c05d38, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0008252b0, {0x2340c20, 0xc000716e80}, 0xc000674720?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc0008bc240, 0xc000c05e58, 0xc000c05ea0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008bc240, {0x2340c20?, 0xc000716e80?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0008be220) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6853 +0x46 +go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000141ae0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:458 +0x26a +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 56 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:301 +0x58 + +goroutine 56 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() + /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 + /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 + +goroutine 22 [select]: +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00080a0f0, 0x1) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0008558f0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 46 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 + +goroutine 44 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b0f4a0, {0x2b2cc60, 0xc000a87d10}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 45 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b0f4d0, {0x2b2cc60, 0xc000a87d60}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 82 [chan receive]: +go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea +created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 59 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c + +goroutine 24 [select]: +go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000141ae0, {0x2b2cc60, 0xc000b34b40}, 0x1f2292329c2c5814, 0xc0000c2720) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 56 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 + +goroutine 52 [select]: +google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000124248) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1694 +0x137 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 46 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:399 +0x1e26 + +goroutine 53 [IO wait]: +internal/poll.runtime_pollWait(0x7f99312c6668, 0x72) + /usr/local/go/src/runtime/netpoll.go:345 +0x85 +internal/poll.(*pollDesc).wait(0xc0005ca080?, 0xc000ac8000?, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0005ca080, {0xc000ac8000, 0x8000, 0x8000}) + /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc0005ca080, {0xc000ac8000?, 0x1060100000000?, 0x8?}) + /usr/local/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc000090510, {0xc000ac8000?, 0x800010601?, 0xc000000000?}) + /usr/local/go/src/net/net.go:185 +0x45 +bufio.(*Reader).Read(0xc0005f8ea0, {0xc000b02200, 0x9, 0x408b620?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc0005f8ea0}, {0xc000b02200, 0x9, 0x9}, 0x9) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b02200, 0x9, 0xc0005b82e8?}, {0x2b0d3e0?, 0xc0005f8ea0?}) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b021c0) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000124248, 0xc0005f8f00) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 46 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 + +goroutine 26 [select]: +go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000141ae0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:550 +0x75 +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 56 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:302 +0x96 + +goroutine 27 [chan receive]: +go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 +created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 56 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 + +goroutine 146 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 130 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 67 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() + /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 + +goroutine 68 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() + /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 + +goroutine 69 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() + /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 66 + /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 + +goroutine 59 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() + /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 69 + /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 + +goroutine 60 [select]: +context.(*cancelCtx).propagateCancel.func2() + /usr/local/go/src/context/context.go:510 +0x98 +created by context.(*cancelCtx).propagateCancel in goroutine 59 + /usr/local/go/src/context/context.go:509 +0x3f3 + +goroutine 61 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000825380) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 +created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 59 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 + +goroutine 63 [chan receive]: +go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea +created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 80 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c + +goroutine 70 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 61 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 71 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0000c1c20, {0xc00091b450, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000c1c20, {0xc00091b450?, 0xc0005b81f8?, 0xc000a099d8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00091b3e0, {0xc00091b450?, 0xc000a09a50?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc00091b3e0}, {0xc00091b450, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008a07e0, {0xc00091b450, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc00091b440, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc00091b440, 0xc0008a07e0, {0x0, 0x0}, 0x7fffffff, 0xc000a09d58, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0xc0000ad460?, {0x0?, 0x0?}, {0x2380420, 0xc00011f030}, 0xc00023ab70?, 0xc000a09d58, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000978270, {0x2380420, 0xc00011f030}, 0x40c573?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc0008a0480, 0xc000a09e78, 0xc000a09ec0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008a0480, {0x2380420?, 0xc00011f030?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc000902620) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000825380, {0x2b39370, 0xc000902620}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 61 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 + +goroutine 72 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000825380, 0xc000968d10, 0xc000813620) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 61 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b + +goroutine 83 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() + /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 + /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 + +goroutine 84 [select]: +go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000141ae0, {0x2b2cc60, 0xc00069f630}, 0x1f2292329c2c581d, 0xc0006b5bc0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 83 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 + +goroutine 85 [chan receive]: +go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 +created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 83 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 + +goroutine 77 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() + /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 + +goroutine 88 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New.func1() + /code/internal/pkg/service/common/servicectx/servicectx.go:100 +0x26 +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New in goroutine 15 + /code/internal/pkg/service/common/servicectx/servicectx.go:98 +0x23c + +goroutine 97 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b86930, {0x2b2cc60, 0xc000b749b0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 76 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() + /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 + /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 + +goroutine 98 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b86960, {0x2b2cc60, 0xc000b74a00}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 99 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b86990, {0x2b2cc60, 0xc000b74a50}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 109 [select]: +go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000b4d860, {0x2b2cc60, 0xc000b75360}, 0x1f2292329c2c5824, 0xc000b67620) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 76 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 + +goroutine 110 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000b75ae0, {0xc000bcd0c0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b75ae0, {0xc000bcd0c0?, 0xc000a9fb00?, 0xc000ac39b8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000bcd050, {0xc000bcd0c0?, 0xc000ac3a30?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc000bcd050}, {0xc000bcd0c0, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000bdc120, {0xc000bcd0c0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc000bcd0b0, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc000bcd0b0, 0xc000bdc120, {0x0, 0x0}, 0x7fffffff, 0xc000ac3d38, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0xc000ac3c98?, {0x7f992fe94558, 0x40f8680}, 0xc000ac3ca0?, {0x0?, 0x0?}, {0x2340c20, 0xc0006b3d80}, 0xc000ac3cb8?, 0xc000ac3d38, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000b731e0, {0x2340c20, 0xc0006b3d80}, 0xc000c24fc0?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc000b7bd40, 0xc000ac3e58, 0xc000ac3ea0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000b7bd40, {0x2340c20?, 0xc0006b3d80?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc000b87970) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6853 +0x46 +go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000b4d860) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:458 +0x26a +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 76 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:301 +0x58 + +goroutine 106 [select]: +google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000bba008) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1694 +0x137 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 100 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:399 +0x1e26 + +goroutine 107 [IO wait]: +internal/poll.runtime_pollWait(0x7f99312c6570, 0x72) + /usr/local/go/src/runtime/netpoll.go:345 +0x85 +internal/poll.(*pollDesc).wait(0xc00067d980?, 0xc000baa000?, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00067d980, {0xc000baa000, 0x8000, 0x8000}) + /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc00067d980, {0xc000baa000?, 0x1060100000000?, 0x8?}) + /usr/local/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc000711ae8, {0xc000baa000?, 0x800010601?, 0xc000000000?}) + /usr/local/go/src/net/net.go:185 +0x45 +bufio.(*Reader).Read(0xc000675d40, {0xc0001afee0, 0x9, 0x408b620?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc000675d40}, {0xc0001afee0, 0x9, 0x9}, 0x9) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0001afee0, 0x9, 0xc000054570?}, {0x2b0d3e0?, 0xc000675d40?}) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001afea0) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000bba008, 0xc000675da0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 100 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 + +goroutine 108 [select]: +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000b75130, 0x1) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00011ecb0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 100 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 + +goroutine 111 [select]: +go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000b4d860) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:550 +0x75 +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 76 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:302 +0x96 + +goroutine 112 [chan receive]: +go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 +created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 76 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 + +goroutine 114 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 110 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 115 [select]: +go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000b4d860, {0x2b39318, 0xc000b87970}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:590 +0x1d2 +created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 110 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:500 +0x2ab + +goroutine 78 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() + /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 + +goroutine 79 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() + /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 + +goroutine 116 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() + /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 77 + /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 + +goroutine 80 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() + /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 116 + /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 + +goroutine 81 [select]: +context.(*cancelCtx).propagateCancel.func2() + /usr/local/go/src/context/context.go:510 +0x98 +created by context.(*cancelCtx).propagateCancel in goroutine 80 + /usr/local/go/src/context/context.go:509 +0x3f3 + +goroutine 130 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000a78f70) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 +created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 80 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 + +goroutine 132 [select]: +go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000b4d860, {0x2b2cc60, 0xc000a513b0}, 0x1f2292329c2c582d, 0xc000c38540) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 162 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 + +goroutine 147 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0009cc320, {0xc0009ca850, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0009cc320, {0xc0009ca850?, 0xc00014c138?, 0xc0004f19d8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0009ca7e0, {0xc0009ca850?, 0xc0004f1a50?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc0009ca7e0}, {0xc0009ca850, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0009e8360, {0xc0009ca850, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc0009ca840, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc0009ca840, 0xc0009e8360, {0x0, 0x0}, 0x7fffffff, 0xc0004f1d58, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0xc0000ad460?, {0x0?, 0x0?}, {0x2380420, 0xc0002a5110}, 0xc00023ab70?, 0xc0004f1d58, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0009ea000, {0x2380420, 0xc0002a5110}, 0x40c573?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc0009e8000, 0xc0004f1e78, 0xc0004f1ec0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009e8000, {0x2380420?, 0xc0002a5110?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc0009d6300) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000a78f70, {0x2b39370, 0xc0009d6300}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 130 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 + +goroutine 148 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000a78f70, 0xc0009f6000, 0xc000c38240) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 130 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b + +goroutine 65 [IO wait]: +internal/poll.runtime_pollWait(0x7f99312c6760, 0x72) + /usr/local/go/src/runtime/netpoll.go:345 +0x85 +internal/poll.(*pollDesc).wait(0xe?, 0xc0009ce900?, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000c96100) + /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac +net.(*netFD).accept(0xc000c96100) + /usr/local/go/src/net/fd_unix.go:172 +0x29 +net.(*TCPListener).accept(0xc0008fa440) + /usr/local/go/src/net/tcpsock_posix.go:159 +0x1e +net.(*TCPListener).Accept(0xc0008fa440) + /usr/local/go/src/net/tcpsock.go:327 +0x30 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/tcp.(*Protocol).Accept(0xc0008c1700?, {0x2b27940?, 0xc0008fa440?}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/tcp/protocol_tcp.go:26 +0x22 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnection(0xc0008c1700, {0x2b2cc28, 0xc000c8c570}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:218 +0x62 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnectionsLoop(0xc0008c1700, {0x2b2cc28, 0xc000c8c570}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:204 +0x67 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.Listen.func1() + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:68 +0x5a +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.Listen in goroutine 15 + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:66 +0x548 + +goroutine 162 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() + /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 + /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 + +goroutine 133 [chan receive]: +go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 +created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 162 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 + +goroutine 165 [semacquire]: +sync.runtime_Semacquire(0x44d74b?) + /usr/local/go/src/runtime/sema.go:62 +0x25 +sync.(*WaitGroup).Wait(0x1cfe4fc?) + /usr/local/go/src/sync/waitgroup.go:116 +0x48 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close(0x2b5c500, {0x2b2cc28, 0xc000d0a6c0}, {0x2401575, 0x8}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:129 +0x22b +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router.New.func1({0x2b2cc28, 0xc000d0a6c0}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/router.go:92 +0x158 +github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New.func1() + /code/internal/pkg/service/common/servicectx/servicectx.go:104 +0x51 +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New in goroutine 15 + /code/internal/pkg/service/common/servicectx/servicectx.go:98 +0x23c + +goroutine 149 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).Accept(0x2b423c0?) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:76 +0x67 +google.golang.org/grpc.(*Server).Serve(0xc000789600, {0x2b2aae0, 0xc0008c1700}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:885 +0x49e +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.(*NetworkFileServer).serve(0xc0008fcaa0, {0x2b2aae0, 0xc0008c1700}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/server.go:121 +0x2a5 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.StartNetworkFileServer.func2(0xc0005a3390) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/server.go:104 +0x2a +github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.(*Process).Add.func1() + /code/internal/pkg/service/common/servicectx/servicectx.go:174 +0x9d +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.(*Process).Add in goroutine 15 + /code/internal/pkg/service/common/servicectx/servicectx.go:172 +0x77 + +goroutine 150 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/node/writernode/diskcleanup.Start.func2() + /code/internal/pkg/service/stream/storage/node/writernode/diskcleanup/diskcleanup.go:100 +0x1ff +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/node/writernode/diskcleanup.Start in goroutine 15 + /code/internal/pkg/service/stream/storage/node/writernode/diskcleanup/diskcleanup.go:89 +0x319 + +goroutine 125 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d7250, {0x2b2cc60, 0xc000be0a50}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 212 [select]: +github.com/hashicorp/yamux.(*Session).keepalive(0xc0009e29c0) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:349 +0x85 +created by github.com/hashicorp/yamux.newSession in goroutine 30 + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:121 +0x4cf + +goroutine 213 [select]: +github.com/hashicorp/yamux.(*Session).AcceptStream(0xc000befe00) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:242 +0x69 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptStream(0xc0008c1700, 0xc000befe00) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:271 +0x2a +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptStreamsLoop(0xc0008c1700, {0x2b2cc28, 0xc000c8c570}, 0xc000befe00) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:257 +0x86 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnection.func1() + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:242 +0xa5 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnection in goroutine 65 + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:240 +0x39b + +goroutine 211 [select]: +github.com/hashicorp/yamux.(*Session).sendLoop(0xc0009e29c0) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:467 +0x11d +github.com/hashicorp/yamux.(*Session).send(0xc0009e29c0) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:456 +0x18 +created by github.com/hashicorp/yamux.newSession in goroutine 30 + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:119 +0x485 + +goroutine 183 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() + /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 161 + /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 + +goroutine 126 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d7280, {0x2b2cc60, 0xc000be0aa0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 158 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000a50640, {0xc000d8d150, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a50640, {0xc000d8d150?, 0xc0006ae468?, 0xc0011099b8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000d8d0e0, {0xc000d8d150?, 0xc001109a30?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc000d8d0e0}, {0xc000d8d150, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008266c0, {0xc000d8d150, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc000d8d140, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc000d8d140, 0xc0008266c0, {0x0, 0x0}, 0x7fffffff, 0xc001109d38, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0xc001109c98?, {0x7f992fe94558, 0x40f8680}, 0xc001109ca0?, {0x0?, 0x0?}, {0x2340c20, 0xc0006b3bc0}, 0xc001109cb8?, 0xc001109d38, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000a781a0, {0x2340c20, 0xc0006b3bc0}, 0xc0005f9f20?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc000826360, 0xc001109e58, 0xc001109ea0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000826360, {0x2340c20?, 0xc0006b3bc0?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc000c20680) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6853 +0x46 +go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000e96c80) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:458 +0x26a +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 156 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:301 +0x58 + +goroutine 127 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d72b0, {0x2b2cc60, 0xc000be0af0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 30 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*ClientConnection).dialLoop(0xc0002a41c0, {0x2b2cc28, 0xc000bcc540}, 0x0) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/client_conn.go:195 +0x67c +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.newClientConnection.func1() + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/client_conn.go:58 +0x73 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.newClientConnection in goroutine 29 + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/client_conn.go:56 +0x5ab + +goroutine 156 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() + /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 + /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 + +goroutine 157 [select]: +go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000e96c80, {0x2b2cc60, 0xc0008fd810}, 0x1f2292329c2c5836, 0xc0000c33e0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 156 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 + +goroutine 153 [select]: +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000be11d0, 0x1) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000f08c0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 128 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 + +goroutine 181 [select]: +google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000688008) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1694 +0x137 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 128 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:399 +0x1e26 + +goroutine 182 [IO wait]: +internal/poll.runtime_pollWait(0x7f99312c6478, 0x72) + /usr/local/go/src/runtime/netpoll.go:345 +0x85 +internal/poll.(*pollDesc).wait(0xc000682580?, 0xc000316000?, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000682580, {0xc000316000, 0x8000, 0x8000}) + /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc000682580, {0xc000316000?, 0x1060100000000?, 0x8?}) + /usr/local/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc000710e18, {0xc000316000?, 0x800010601?, 0xc000000000?}) + /usr/local/go/src/net/net.go:185 +0x45 +bufio.(*Reader).Read(0xc000bc9620, {0xc0001af380, 0x9, 0xc000093008?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc000bc9620}, {0xc0001af380, 0x9, 0x9}, 0x9) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0001af380, 0x9, 0xc0006ae480?}, {0x2b0d3e0?, 0xc000bc9620?}) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001af340) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000688008, 0xc000bc9680) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 128 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 + +goroutine 159 [select]: +go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000e96c80) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:550 +0x75 +created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 156 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:302 +0x96 + +goroutine 160 [chan receive]: +go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 +created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 156 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 + +goroutine 167 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 158 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 168 [select]: +go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000e96c80, {0x2b39318, 0xc000c20680}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:590 +0x1d2 +created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 158 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:500 +0x2ab + +goroutine 134 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() + /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 + +goroutine 135 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() + /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 + +goroutine 136 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() + /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 + +goroutine 161 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() + /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 134 + /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 + +goroutine 184 [select]: +context.(*cancelCtx).propagateCancel.func2() + /usr/local/go/src/context/context.go:510 +0x98 +created by context.(*cancelCtx).propagateCancel in goroutine 183 + /usr/local/go/src/context/context.go:509 +0x3f3 + +goroutine 185 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc0009eb450) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 +created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 183 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 + +goroutine 210 [IO wait]: +internal/poll.runtime_pollWait(0x7f99312c6380, 0x72) + /usr/local/go/src/runtime/netpoll.go:345 +0x85 +internal/poll.(*pollDesc).wait(0xc0009e6780?, 0xc000268000?, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0009e6780, {0xc000268000, 0x1000, 0x1000}) + /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc0009e6780, {0xc000268000?, 0x2b0d3e0?, 0xc0009db620?}) + /usr/local/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc0009e4070, {0xc000268000?, 0xc000093008?, 0xc000d00720?}) + /usr/local/go/src/net/net.go:185 +0x45 +bufio.(*Reader).Read(0xc0009db620, {0xc0005b4f70, 0xc, 0x0?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc0009db620}, {0xc0005b4f70, 0xc, 0xc}, 0xc) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +github.com/hashicorp/yamux.(*Session).recvLoop(0xc0009e29c0) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:536 +0xb9 +github.com/hashicorp/yamux.(*Session).recv(0xc0009e29c0) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:515 +0x18 +created by github.com/hashicorp/yamux.newSession in goroutine 30 + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:118 +0x446 + +goroutine 187 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 185 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 188 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000be1a90, {0xc000c8dd80, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000be1a90, {0xc000c8dd80?, 0xc000a9ec78?, 0xc000bf79d8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c8dd10, {0xc000c8dd80?, 0xc000bf7a50?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc000c8dd10}, {0xc000c8dd80, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0004d3560, {0xc000c8dd80, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc000c8dd70, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc000c8dd70, 0xc0004d3560, {0x0, 0x0}, 0x7fffffff, 0xc000bf7d58, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0x0?, {0x0?, 0x0?}, {0x2380420, 0xc00011f0a0}, 0xc000bc9620?, 0xc000bf7d58, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0009eb6c0, {0x2380420, 0xc00011f0a0}, 0x40c573?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc0009e9d40, 0xc000bf7e78, 0xc000bf7ec0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009e9d40, {0x2380420?, 0xc00011f0a0?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc000850260) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc0009eb450, {0x2b39370, 0xc000850260}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 185 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 + +goroutine 189 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0009eb450, 0xc0009f62c0, 0xc000be9620) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 185 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b + +goroutine 194 [chan receive]: +go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea +created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 183 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c + +goroutine 195 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() + /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 + +goroutine 196 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() + /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 + +goroutine 197 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() + /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 + +goroutine 169 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() + /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 195 + /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 + +goroutine 170 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() + /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 169 + /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 + +goroutine 171 [select]: +context.(*cancelCtx).propagateCancel.func2() + /usr/local/go/src/context/context.go:510 +0x98 +created by context.(*cancelCtx).propagateCancel in goroutine 170 + /usr/local/go/src/context/context.go:509 +0x3f3 + +goroutine 172 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000a78410) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 +created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 170 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 + +goroutine 174 [chan receive]: +go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea +created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 235 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c + +goroutine 198 [IO wait]: +internal/poll.runtime_pollWait(0x7f99312c6288, 0x72) + /usr/local/go/src/runtime/netpoll.go:345 +0x85 +internal/poll.(*pollDesc).wait(0xc000c47900?, 0xc000b4b000?, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000c47900, {0xc000b4b000, 0x1000, 0x1000}) + /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc000c47900, {0xc000b4b000?, 0x2b0d3e0?, 0xc0006740c0?}) + /usr/local/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc0005ce550, {0xc000b4b000?, 0x40cab0?, 0xc000c84e60?}) + /usr/local/go/src/net/net.go:185 +0x45 +bufio.(*Reader).Read(0xc0006740c0, {0xc0009ddcb0, 0xc, 0x0?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc0006740c0}, {0xc0009ddcb0, 0xc, 0xc}, 0xc) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +github.com/hashicorp/yamux.(*Session).recvLoop(0xc000befe00) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:536 +0xb9 +github.com/hashicorp/yamux.(*Session).recv(0xc000befe00) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:515 +0x18 +created by github.com/hashicorp/yamux.newSession in goroutine 65 + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:118 +0x446 + +goroutine 199 [select]: +github.com/hashicorp/yamux.(*Session).sendLoop(0xc000befe00) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:467 +0x11d +github.com/hashicorp/yamux.(*Session).send(0xc000befe00) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:456 +0x18 +created by github.com/hashicorp/yamux.newSession in goroutine 65 + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:119 +0x485 + +goroutine 200 [select]: +github.com/hashicorp/yamux.(*Session).keepalive(0xc000befe00) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:349 +0x85 +created by github.com/hashicorp/yamux.newSession in goroutine 65 + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:121 +0x4cf + +goroutine 226 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 172 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 227 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000a86550, {0xc0007ac2e0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a86550, {0xc0007ac2e0?, 0xc000054318?, 0xc0007539d8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0007ac270, {0xc0007ac2e0?, 0xc000753a50?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc0007ac270}, {0xc0007ac2e0, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000afe360, {0xc0007ac2e0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc0007ac2d0, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc0007ac2d0, 0xc000afe360, {0x0, 0x0}, 0x7fffffff, 0xc000753d58, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0x0?, {0x0?, 0x0?}, {0x2380420, 0xc00029c070}, 0x0?, 0xc000753d58, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000aaa1a0, {0x2380420, 0xc00029c070}, 0x40c573?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc000afe000, 0xc000753e78, 0xc000753ec0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000afe000, {0x2380420?, 0xc00029c070?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc00088e9b0) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000a78410, {0x2b39370, 0xc00088e9b0}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 172 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 + +goroutine 228 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000a78410, 0xc00085e2c0, 0xc000c38fc0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 172 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b + +goroutine 229 [chan receive]: +go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea +created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 170 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c + +goroutine 230 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/statistics/collector.Start.func4() + /code/internal/pkg/service/stream/storage/statistics/collector/collector.go:128 +0x115 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/statistics/collector.Start in goroutine 15 + /code/internal/pkg/service/stream/storage/statistics/collector/collector.go:122 +0x445 + +goroutine 231 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() + /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 + +goroutine 232 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() + /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x253 +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 + +goroutine 233 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() + /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 + /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 + +goroutine 234 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() + /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 231 + /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 + +goroutine 235 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() + /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca +created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 234 + /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 + +goroutine 236 [select]: +context.(*cancelCtx).propagateCancel.func2() + /usr/local/go/src/context/context.go:510 +0x98 +created by context.(*cancelCtx).propagateCancel in goroutine 235 + /usr/local/go/src/context/context.go:509 +0x3f3 + +goroutine 237 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000aaa4e0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 +created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 235 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 + +goroutine 239 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 237 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 240 [select]: +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000a87680, {0xc0007adcf0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a87680, {0xc0007adcf0?, 0xc000054648?, 0xc0008759d8?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0007adc80, {0xc0007adcf0?, 0xc000875a50?, 0xbeb985?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c +io.ReadAtLeast({0x2b11be0, 0xc0007adc80}, {0xc0007adcf0, 0x5, 0x5}, 0x5) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000aff9e0, {0xc0007adcf0, 0x5, 0x5}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 +google.golang.org/grpc.(*parser).recvMsg(0xc0007adce0, 0x7fffffff) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 +google.golang.org/grpc.recvAndDecompress(0xc0007adce0, 0xc000aff9e0, {0x0, 0x0}, 0x7fffffff, 0xc000875d58, {0x0, 0x0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 +google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0x0?, {0x0?, 0x0?}, {0x2380420, 0xc00029c310}, 0x0?, 0xc000875d58, ...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000aaac30, {0x2380420, 0xc00029c310}, 0x40c573?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc000afed80, 0xc000875e78, 0xc000875ec0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000afed80, {0x2380420?, 0xc00029c310?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc00088f590) + /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000aaa4e0, {0x2b39370, 0xc00088f590}) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 237 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 + +goroutine 241 [select]: +go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000aaa4e0, 0xc00085ea50, 0xc0006b59e0) + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e +created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 237 + /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b + +goroutine 279 [sync.RWMutex.Lock]: +sync.runtime_SemacquireRWMutex(0x1?, 0x1?, 0x1ed9120?) + /usr/local/go/src/runtime/sema.go:87 +0x25 +sync.(*RWMutex).Lock(0xc0010f0280?) + /usr/local/go/src/sync/rwmutex.go:151 +0x6a +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SlicePipeline).Close(0xc001060630, {0x2b2cc60, 0xc00080ab90}, {0x2401575, 0x8}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_slice.go:129 +0x65 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close.func1() + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:126 +0x71 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close in goroutine 278 + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:124 +0x15c + +goroutine 278 [semacquire]: +sync.runtime_Semacquire(0x44d74b?) + /usr/local/go/src/runtime/sema.go:62 +0x25 +sync.(*WaitGroup).Wait(0x1cfa95c?) + /usr/local/go/src/sync/waitgroup.go:116 +0x48 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close(0x2b5c680, {0x2b2ccd0, 0xc0002a6a80}, {0x2401575, 0x8}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:129 +0x22b +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SinkPipeline).close(0xc00099fe00, {0x2b2ccd0, 0xc0002a6a80}, {0x2401575, 0x8}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_sink.go:190 +0x114 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SinkPipeline).Close(0xc00099fe00, {0x2b2ccd0, 0xc0002a6a80}, {0x2401575, 0x8}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_sink.go:177 +0xd4 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close.func1() + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:126 +0x71 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close in goroutine 165 + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:124 +0x15c + +goroutine 266 [semacquire]: +sync.runtime_Semacquire(0xc000c2bd80?) + /usr/local/go/src/runtime/sema.go:62 +0x25 +sync.(*WaitGroup).Wait(0xc000e80ea0?) + /usr/local/go/src/sync/waitgroup.go:116 +0x48 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).DispatchToSource(0xc000ab0360, {{0x2b0de40?, 0xc000aa8ed0?}, {0x2402fcb?, 0x61?}}, {0x2b3a1e0, 0xc0006ba420}) + /code/internal/pkg/service/stream/sink/router/router.go:255 +0x2b7 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testcase.(*WriterTestCase).Run.func4() + /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:136 +0xc5 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testcase.(*WriterTestCase).Run in goroutine 15 + /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:133 +0x1130 + +goroutine 267 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync/notify.(*Notifier).Wait(0xc00113a4b0, {0x2b2c7e0, 0x40f8680}) + /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/notify/notify.go:37 +0x7e +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.(*pipeline).WriteRecord(0xc0010f4c60, {0x2b3a1e0, 0xc0006ba420}) + /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:289 +0x24c +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SlicePipeline).WriteRecord(0xc001060630, {0x2b3a1e0, 0xc0006ba420}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_slice.go:114 +0xe8 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/balancer.RoundRobinBalancer.WriteRecord({0x1?}, {0x2b3a1e0?, 0xc0006ba420?}, {0xc0011389c0?, 0x6f?, 0xc000e403c0?}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/balancer/balancer_rr.go:28 +0x77 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SinkPipeline).WriteRecord(0xc000704a90?, {0x2b3a1e0?, 0xc0006ba420?}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_sink.go:98 +0xe3 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*pipelineRef).writeRecord(0xc000704a90, {0x2b3a1e0, 0xc0006ba420}) + /code/internal/pkg/service/stream/sink/router/pipeline.go:55 +0x62 +github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).writeRecord(0xc000ab0360, 0xc000a877c0, {0x2b3a1e0, 0xc0006ba420}) + /code/internal/pkg/service/stream/sink/router/router.go:314 +0x6e +github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).dispatchToSink(0xc000ab0360, 0xc000a877c0, {0x2b3a1e0, 0xc0006ba420}) + /code/internal/pkg/service/stream/sink/router/router.go:283 +0x8b +github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).DispatchToSource.func1() + /code/internal/pkg/service/stream/sink/router/router.go:234 +0xb4 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).DispatchToSource in goroutine 266 + /code/internal/pkg/service/stream/sink/router/router.go:230 +0x2a5 + +goroutine 269 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010ab6f0, {0x2b2cc60, 0xc0010a9c20}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 268 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 270 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010ab720, {0x2b2cc60, 0xc0010a9cc0}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 268 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 271 [select]: +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010ab750, {0x2b2cc60, 0xc0010a9d10}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 268 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a + +goroutine 277 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.(*NetworkFileServer).KeepAliveStream(0xc0008fcaa0, 0xc0006578c0, {0x2b38160, 0xc000ec8130}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/server.go:157 +0xaa +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/pb._NetworkFile_KeepAliveStream_Handler({0x22d8ca0, 0xc0008fcaa0}, {0x2b33e40, 0xc0010b4000}) + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/pb/networkFile_grpc.pb.go:185 +0x107 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000789600, {0x2b2cc28, 0xc0006576e0}, {0x2b3a360, 0xc000b5c780}, 0xc0001a4120, 0xc0007a7560, 0x3dfaee0, 0x0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1673 +0x1208 +google.golang.org/grpc.(*Server).handleStream(0xc000789600, {0x2b3a360, 0xc000b5c780}, 0xc0001a4120) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1794 +0xe3a +google.golang.org/grpc.(*Server).serveStreams.func2.1() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1029 +0x8b +created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 207 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1040 +0x125 + +goroutine 276 [select]: +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001102320, 0x1) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00011f420) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 272 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 + +goroutine 275 [select]: +github.com/hashicorp/yamux.(*Stream).Read(0xc0010fe5b0, {0xc001112000, 0x8000, 0xc000c85d30?}) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/stream.go:145 +0x40e +bufio.(*Reader).Read(0xc000c272c0, {0xc000f4d7e0, 0x9, 0x408b620?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc000c272c0}, {0xc000f4d7e0, 0x9, 0x9}, 0x9) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f4d7e0, 0x9, 0x21290a0?}, {0x2b0d3e0?, 0xc000c272c0?}) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f4d7a0) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000689448, 0xc000c27320) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 272 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 + +goroutine 205 [select]: +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00080b360, 0x1) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00011f3b0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:334 +0xc7 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 204 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:332 +0x193e + +goroutine 206 [select]: +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b5c780) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:1168 +0x205 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 204 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:355 +0x1985 + +goroutine 207 [select]: +github.com/hashicorp/yamux.(*Stream).Read(0xc0008251e0, {0xc000eb8000, 0x8000, 0x0?}) + /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/stream.go:145 +0x40e +bufio.(*Reader).Read(0xc000675c80, {0xc000a1e2e0, 0x9, 0x408b620?}) + /usr/local/go/src/bufio/bufio.go:241 +0x197 +io.ReadAtLeast({0x2b0d3e0, 0xc000675c80}, {0xc000a1e2e0, 0x9, 0x9}, 0x9) + /usr/local/go/src/io/io.go:335 +0x90 +io.ReadFull(...) + /usr/local/go/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000a1e2e0, 0x9, 0xc0005b8120?}, {0x2b0d3e0?, 0xc000675c80?}) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000a1e2a0) + /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b5c780, {0x2b2cc28, 0xc000c146f0}, 0xc000c14780) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:644 +0x15e +google.golang.org/grpc.(*Server).serveStreams(0xc000789600, {0x2b2c7e0?, 0x40f8680?}, {0x2b3a360, 0xc000b5c780}, {0x2b39840?, 0xc000da3c60?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1023 +0x3b6 +google.golang.org/grpc.(*Server).handleRawConn.func1() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:959 +0x56 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 204 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:958 +0x1c6 + +goroutine 137 [select]: +google.golang.org/grpc.newClientStreamWithParams.func4() + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 268 + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 + +goroutine 138 [select]: +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00103a240) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:330 +0x7c +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:345 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0010ff5f0, {0x21553c0, 0xc000c14030}, 0x0?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1072 +0xc9 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x30?) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f +google.golang.org/grpc.(*clientStream).withRetry(0xc00103a000, 0xc000eb6e80, 0xc000eb6ec8) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:778 +0x13a +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00103a000, {0x21553c0?, 0xc000c14030?}) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 +google.golang.org/grpc.(*GenericClientStream[...]).Recv(0x2b3b0c0) + /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream_interfaces.go:99 +0x4e +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.OpenNetworkFile.func2() + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/client.go:124 +0x62 +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.OpenNetworkFile in goroutine 268 + /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/client.go:122 +0x925 + +goroutine 139 [chan receive]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.(*pipeline).processChunks(0xc0010f4c60, {0x2b2cc28, 0xc0010fdaa0}, {0x2b3b3f0, 0x40f8680}, {{{0xc00013b79c, 0x3}, 0x0, 0x180000, {0x0, ...}}, ...}) + /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:418 +0x6c +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.newPipeline.func2() + /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:165 +0xbe +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.newPipeline in goroutine 268 + /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:163 +0x72e + +goroutine 140 [select]: +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).syncLoop.func1() + /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:242 +0xea +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).syncLoop in goroutine 268 + /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:236 +0x98 + +goroutine 177 [sync.RWMutex.Lock]: +sync.runtime_SemacquireRWMutex(0xc000eb7de8?, 0xff?, 0xc1b5ba829e8d0880?) + /usr/local/go/src/runtime/sema.go:87 +0x25 +sync.(*RWMutex).Lock(0x0?) + /usr/local/go/src/sync/rwmutex.go:151 +0x6a +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.(*pipeline).Flush(0xc0010f4c60, {0x2b2c7e0, 0x40f8680}) + /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:339 +0x5c +github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).TriggerSync.func1() + /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:184 +0x17e +created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).TriggerSync in goroutine 140 + /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:176 +0x225 +FAIL github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/encoder/csv 10.062s +FAIL From e495015123466fcc3a70db01971f2a55dd0ec50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 27 Sep 2024 11:05:46 +0200 Subject: [PATCH 5/5] fix: Remove debug txt file --- out.txt | 1469 ------------------------------------------------------- 1 file changed, 1469 deletions(-) delete mode 100644 out.txt diff --git a/out.txt b/out.txt deleted file mode 100644 index 844ca48e35..0000000000 --- a/out.txt +++ /dev/null @@ -1,1469 +0,0 @@ -? github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/encoder/csv/fastcsv [no test files] -=== RUN TestCSVWriter -=== PAUSE TestCSVWriter -=== RUN TestCSVWriterAboveLimit -=== PAUSE TestCSVWriterAboveLimit -=== CONT TestCSVWriter -=== CONT TestCSVWriterAboveLimit -=== RUN TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false -=== PAUSE TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false ---- PASS: TestCSVWriterAboveLimit (0.00s) -=== CONT TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false -{"level":"info","time":"2024-09-27T08:53:29.429Z","message":"connecting to etcd","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} -{"level":"info","time":"2024-09-27T08:53:29.490Z","message":"connected to etcd cluster \"etcd:2379\"","duration":"61.351095ms","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} -{"level":"info","time":"2024-09-27T08:53:29.493Z","message":"creating etcd session","component":"distribution.mutex.provider.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.556Z","message":"created etcd session","duration":"63.125746ms","component":"distribution.mutex.provider.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.616Z","message":"watch stream created","nodeId":"api","stream.prefix":"storage/volume/writer/","component":"volume.repository"} -{"level":"info","time":"2024-09-27T08:53:29.616Z","message":"node ID \"api\"","nodeId":"api","node":"api","component":"task"} -{"level":"info","time":"2024-09-27T08:53:29.616Z","message":"creating etcd session","nodeId":"api","node":"api","component":"task.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.678Z","message":"created etcd session","nodeId":"api","duration":"61.823622ms","node":"api","component":"task.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.679Z","message":"connecting to etcd","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} -{"level":"info","time":"2024-09-27T08:53:29.737Z","message":"connected to etcd cluster \"etcd:2379\"","duration":"58.314469ms","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} -{"level":"info","time":"2024-09-27T08:53:29.738Z","message":"creating etcd session","component":"distribution.mutex.provider.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.798Z","message":"created etcd session","duration":"60.116063ms","component":"distribution.mutex.provider.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.857Z","message":"watch stream created","nodeId":"disk-writer","stream.prefix":"storage/volume/writer/","component":"volume.repository"} -{"level":"info","time":"2024-09-27T08:53:29.857Z","message":"searching for volumes in volumes path","nodeId":"disk-writer","volumes.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002","component":"storage.node.writer.volumes"} -{"level":"info","time":"2024-09-27T08:53:29.857Z","message":"found volume","nodeId":"disk-writer","volume.label":"001","volume.type":"hdd","component":"storage.node.writer.volumes"} -{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"opening volume","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","component":"storage.node.writer.volumes.volume"} -{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"opened volume","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.type":"hdd","volume.label":"001","component":"storage.node.writer.volumes.volume"} -{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"found \"1\" volumes","nodeId":"disk-writer","component":"storage.node.writer.volumes"} -{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"starting storage writer node","nodeId":"disk-writer","component":"storage.node.writer"} -{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"disk writer listening on \"[::]:33631\"","nodeId":"disk-writer","listenAddress":"0.0.0.0:33631","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} -{"level":"info","time":"2024-09-27T08:53:29.858Z","message":"creating etcd session","nodeId":"disk-writer","component":"volumes.registry.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:29.925Z","message":"created etcd session","nodeId":"disk-writer","duration":"66.514211ms","component":"volumes.registry.etcd.session"} -{"level":"debug","time":"2024-09-27T08:53:29.926Z","message":"watch stream mirror synced to revision 1357","nodeId":"api","stream.prefix":"storage/volume/writer/","component":"volume.repository"} -{"level":"debug","time":"2024-09-27T08:53:29.926Z","message":"watch stream mirror synced to revision 1357","nodeId":"disk-writer","stream.prefix":"storage/volume/writer/","component":"volume.repository"} -{"level":"info","time":"2024-09-27T08:53:29.927Z","message":"registered \"1\" volumes","nodeId":"disk-writer","component":"volumes.registry"} -{"level":"info","time":"2024-09-27T08:53:29.927Z","message":"removing expired files without DB record from disk","nodeId":"disk-writer","component":"storage.disk.cleanup"} -{"level":"info","time":"2024-09-27T08:53:29.928Z","message":"connecting to etcd","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} -{"level":"info","time":"2024-09-27T08:53:29.928Z","message":"removed \"0\" directories","nodeId":"disk-writer","removedDirectoriesCount":0,"component":"storage.disk.cleanup"} -{"level":"info","time":"2024-09-27T08:53:29.988Z","message":"connected to etcd cluster \"etcd:2379\"","duration":"59.437793ms","etcd.connect.timeout":"30s","etcd.endpoints":["etcd:2379"],"etcd.keepAlive.interval":"10s","etcd.keepAlive.timeout":"5s","component":"etcd.client"} -{"level":"info","time":"2024-09-27T08:53:29.989Z","message":"creating etcd session","component":"distribution.mutex.provider.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:30.049Z","message":"created etcd session","duration":"59.768429ms","component":"distribution.mutex.provider.etcd.session"} -{"level":"debug","time":"2024-09-27T08:53:30.049Z","message":"watch stream mirror synced to revision 1357","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"volume.repository"} -{"level":"info","time":"2024-09-27T08:53:30.107Z","message":"watch stream created","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"volume.repository"} -{"level":"debug","time":"2024-09-27T08:53:30.107Z","message":"watch stream mirror synced to revision 1357","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"storage.router.connections"} -{"level":"info","time":"2024-09-27T08:53:30.107Z","message":"the list of volumes has changed, updating connections","nodeId":"source","component":"storage.router.connections"} -{"level":"debug","time":"2024-09-27T08:53:30.107Z","message":"new disk writer node \"disk-writer\"","nodeId":"source","component":"storage.router.connections"} -{"level":"debug","time":"2024-09-27T08:53:30.107Z","message":"registered connection to \"disk-writer\", total connections count 1","nodeId":"source","nodeId":"source","remoteAddr":"localhost:33631","remoteNodeID":"disk-writer","component":"storage.router.connections.client.transport"} -{"level":"info","time":"2024-09-27T08:53:30.107Z","message":"disk writer client is connecting to \"disk-writer\" - \"localhost:33631\"","nodeId":"source","nodeId":"source","remoteAddr":"localhost:33631","remoteNodeID":"disk-writer","component":"storage.router.connections.client.transport"} -{"level":"info","time":"2024-09-27T08:53:30.108Z","message":"disk writer client connected from \"127.0.0.1:54770\" to \"disk-writer\" - \"localhost:33631\"","nodeId":"source","nodeId":"source","remoteAddr":"localhost:33631","remoteNodeID":"disk-writer","component":"storage.router.connections.client.transport"} -{"level":"info","time":"2024-09-27T08:53:30.108Z","message":"accepted connection from \"127.0.0.1:54770\" to \"127.0.0.1:33631\"","nodeId":"disk-writer","listenAddress":"0.0.0.0:33631","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} -{"level":"debug","time":"2024-09-27T08:53:30.108Z","message":"registered session to \"127.0.0.1:54770\", total sessions count 1","nodeId":"disk-writer","listenAddress":"0.0.0.0:33631","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} -{"level":"info","time":"2024-09-27T08:53:30.165Z","message":"watch stream created","nodeId":"source","stream.prefix":"storage/volume/writer/","component":"storage.router.connections"} -{"level":"info","time":"2024-09-27T08:53:30.225Z","message":"watch stream created","nodeId":"source","stream.prefix":"definition/sink/active/","component":"sink.router"} -{"level":"info","time":"2024-09-27T08:53:30.225Z","message":"joining distribution group","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:30.225Z","message":"creating etcd session","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:30.284Z","message":"created etcd session","nodeId":"source","duration":"59.522845ms","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:30.284Z","message":"registering the node \"source\"","nodeId":"source","node":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:30.286Z","message":"the node \"source\" registered","nodeId":"source","duration":"1.177936ms","node":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:30.286Z","message":"watching for other nodes","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:30.286Z","message":"found a new node \"source\"","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:30.344Z","message":"watch stream created","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","stream.prefix":"","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:30.344Z","message":"creating etcd session","nodeId":"source","component":"close-sync.source.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:30.401Z","message":"created etcd session","nodeId":"source","duration":"57.140794ms","component":"close-sync.source.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:30.460Z","message":"watch stream created","nodeId":"source","stream.prefix":"storage/slice/level/local/","component":"storage.router"} -{"level":"debug","time":"2024-09-27T08:53:30.472Z","message":"executing OnBranchSave \"123/111\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.474Z","message":"executing OnSourceSave \"123/111/my-source\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.478Z","message":"executing OnSinkSave \"123/111/my-source/my-sink\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.479Z","message":"executing OnFileOpen \"123/111/my-source/my-sink\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.479Z","message":"executing OnFileSave \"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.480Z","message":"executing OnSliceOpen \"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.480Z","message":"executing OnSliceSave \"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z\"","nodeId":"api","component":"plugin"} -{"level":"debug","time":"2024-09-27T08:53:30.484Z","message":"watch stream mirror synced to revision 1362","nodeId":"source","component":"sink.router"} -{"level":"debug","time":"2024-09-27T08:53:30.484Z","message":"watch stream mirror synced to revision 1362","nodeId":"source","stream.prefix":"storage/slice/level/local/","component":"storage.router"} -{"level":"info","time":"2024-09-27T08:53:30.498Z","message":"opening sink pipeline","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","branch.id":"111","project.id":"123","sink.id":"my-sink","source.id":"my-source","component":"sink.router"} -{"level":"debug","time":"2024-09-27T08:53:30.499Z","message":"registered stream \"1\" to \"127.0.0.1:54770\", total streams count 1","nodeId":"disk-writer","component":"storage.node.writer.rpc.transport"} -{"level":"debug","time":"2024-09-27T08:53:30.500Z","message":"opening disk writer","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","component":"storage.node.writer.volumes.volume"} -{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"opened file","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","file.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001/123/111/my-source/my-sink/2024-09-27T08-53-30-477Z/2024-09-27T08-53-30-477Z/slice-source.csv","component":"storage.node.writer.volumes.volume"} -{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"allocated disk space \"100MB\"","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","file.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001/123/111/my-source/my-sink/2024-09-27T08-53-30-477Z/2024-09-27T08-53-30-477Z/slice-source.csv","component":"storage.node.writer.volumes.volume"} -{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"opened disk writer","nodeId":"disk-writer","volume.path":"/tmp/TestCSVWritercompression-none-sync-cache-wait-true-parallel-false523292083/002/hdd/001","volume.id":"my-volume-001","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","sourceNode.id":"source","component":"storage.node.writer.volumes.volume"} -{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"opening encoding pipeline","nodeId":"source","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice":"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer","component":"encoding.pipeline"} -{"level":"debug","time":"2024-09-27T08:53:30.501Z","message":"sync is enabled, mode=cache, sync each {count=5000 or uncompressed=10MB or compressed=1MB or interval=10ms}, check each 5ms","nodeId":"source","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice":"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer"} -{"level":"debug","time":"2024-09-27T08:53:30.502Z","message":"opened encoding pipeline","nodeId":"source","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice":"123/111/my-source/my-sink/2024-09-27T08:53:30.477Z/my-volume-001/2024-09-27T08:53:30.477Z","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer","component":"encoding.pipeline"} -{"level":"info","time":"2024-09-27T08:53:30.502Z","message":"opened slice pipeline","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","file.id":"2024-09-27T08:53:30.477Z","volume.id":"my-volume-001","slice.id":"2024-09-27T08:53:30.477Z","branch.id":"111","file.id":"2024-09-27T08:53:30.477Z","project.id":"123","sink.id":"my-sink","slice.id":"2024-09-27T08:53:30.477Z","source.id":"my-source","volume.id":"my-volume-001","writerNodeAddress":"localhost:33631","writerNodeId":"disk-writer","component":"storage.router"} -{"level":"info","time":"2024-09-27T08:53:30.502Z","message":"opened sink pipeline to 1 slices","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","branch.id":"111","project.id":"123","sink.id":"my-sink","source.id":"my-source","component":"storage.router"} -{"level":"info","time":"2024-09-27T08:53:30.502Z","message":"opened sink pipeline","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","branch.id":"111","project.id":"123","sink.id":"my-sink","source.id":"my-source","component":"sink.router"} -{"level":"debug","time":"2024-09-27T08:53:30.512Z","message":"starting sync to cache","nodeId":"source"} -{"level":"debug","time":"2024-09-27T08:53:31.182Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:32.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} - testcase.go:144: - Error Trace: /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:144 - Error: timeout when waiting for batch 102 - Test: TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false -{"level":"info","time":"2024-09-27T08:53:32.499Z","message":"exiting (test cleanup)"} -{"level":"info","time":"2024-09-27T08:53:32.499Z","message":"closing close-sync source node","nodeId":"source","component":"close-sync.source"} -{"level":"info","time":"2024-09-27T08:53:32.499Z","message":"closing etcd session: context canceled","nodeId":"source","component":"close-sync.source.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"closed etcd session","nodeId":"source","duration":"15.671847ms","component":"close-sync.source.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"closed close-sync source node","nodeId":"source","component":"close-sync.source"} -{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"received shutdown request","nodeId":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"unregistering the node \"source\"","nodeId":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:32.515Z","message":"watch stream consumer closed: context canceled","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","stream.prefix":"","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:32.519Z","message":"the node \"source\" unregistered","nodeId":"source","duration":"4.544208ms","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:32.519Z","message":"closing etcd session: context canceled","nodeId":"source","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"closed etcd session","nodeId":"source","duration":"1.473996ms","distribution.group":"storage.router.sources.test-source","distribution.node":"source","component":"distribution.etcd.session"} -{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"shutdown done","nodeId":"source","component":"distribution"} -{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"closing storage router","nodeId":"source","component":"storage.router"} -{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"watch stream consumer closed: context canceled","nodeId":"source","stream.prefix":"storage/slice/level/local/","component":"storage.router"} -{"level":"info","time":"2024-09-27T08:53:32.521Z","message":"closing 1 sink pipelines","nodeId":"source","component":"storage.router"} -{"level":"debug","time":"2024-09-27T08:53:32.521Z","message":"closing sink pipeline: shutdown","nodeId":"source","project.id":"123","branch.id":"111","source.id":"my-source","sink.id":"my-sink","component":"storage.router"} -{"level":"debug","time":"2024-09-27T08:53:33.166Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:34.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:35.166Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:36.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:37.166Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:38.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -{"level":"debug","time":"2024-09-27T08:53:39.165Z","message":"sync done","nodeId":"source","component":"statistics.collector"} -panic: test timed out after 10s -running tests: - TestCSVWriter/compression-none-sync-cache-wait-true-parallel-false (10s) - -goroutine 306 [running]: -testing.(*M).startAlarm.func1() - /usr/local/go/src/testing/testing.go:2366 +0x385 -created by time.goFunc - /usr/local/go/src/time/sleep.go:177 +0x2d - -goroutine 1 [chan receive]: -testing.tRunner.func1() - /usr/local/go/src/testing/testing.go:1650 +0x4ab -testing.tRunner(0xc000aa69c0, 0xc0009bfc70) - /usr/local/go/src/testing/testing.go:1695 +0x134 -testing.runTests(0xc000aa8828, {0x3e073c0, 0x2, 0x2}, {0x1?, 0x51fa6e?, 0x4089a40?}) - /usr/local/go/src/testing/testing.go:2159 +0x445 -testing.(*M).Run(0xc000a88e60) - /usr/local/go/src/testing/testing.go:2027 +0x68b -main.main() - _testmain.go:53 +0x16c - -goroutine 11 [select]: -go.opencensus.io/stats/view.(*worker).start(0xc0005cad00) - /tmp/cache/go-mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x9f -created by go.opencensus.io/stats/view.init.0 in goroutine 1 - /tmp/cache/go-mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x8d - -goroutine 66 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() - /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 - -goroutine 13 [chan receive]: -testing.tRunner.func1() - /usr/local/go/src/testing/testing.go:1650 +0x4ab -testing.tRunner(0xc000aa6b60, 0x24cc220) - /usr/local/go/src/testing/testing.go:1695 +0x134 -created by testing.(*T).Run in goroutine 1 - /usr/local/go/src/testing/testing.go:1742 +0x390 - -goroutine 58 [select]: -go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000141ae0, {0x2b39318, 0xc0008be220}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:590 +0x1d2 -created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 25 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:500 +0x2ab - -goroutine 15 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.(*Process).WaitForShutdown(...) - /code/internal/pkg/service/common/servicectx/servicectx.go:194 -github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.NewForTest.func1() - /code/internal/pkg/service/common/servicectx/servicectx.go:143 +0xe7 -testing.(*common).Cleanup.func1() - /usr/local/go/src/testing/testing.go:1175 +0x10f -testing.(*common).runCleanup(0xc000aa6ea0, 0x0?) - /usr/local/go/src/testing/testing.go:1353 +0xdb -testing.tRunner.func2() - /usr/local/go/src/testing/testing.go:1683 +0x25 -runtime.Goexit() - /usr/local/go/src/runtime/panic.go:626 +0x5e -testing.(*common).FailNow(0xc000aa6ea0) - /usr/local/go/src/testing/testing.go:1005 +0x4a -github.com/stretchr/testify/require.Fail({0x2b19b88, 0xc000aa6ea0}, {0xc00106e150, 0x22}, {0x0, 0x0, 0x0}) - /tmp/cache/go-mod/github.com/stretchr/testify@v1.9.0/require/require.go:508 +0xca -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testcase.(*WriterTestCase).Run(0xc000a8b180, 0xc000aa6ea0) - /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:144 +0x126b -testing.tRunner(0xc000aa6ea0, 0xc00088f400) - /usr/local/go/src/testing/testing.go:1689 +0xfb -created by testing.(*T).Run in goroutine 13 - /usr/local/go/src/testing/testing.go:1742 +0x390 - -goroutine 34 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New.func1() - /code/internal/pkg/service/common/servicectx/servicectx.go:100 +0x26 -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New in goroutine 15 - /code/internal/pkg/service/common/servicectx/servicectx.go:98 +0x23c - -goroutine 43 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b0f470, {0x2b2cc60, 0xc000a87cc0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 57 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 25 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 25 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00013f2c0, {0xc0008bafd0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00013f2c0, {0xc0008bafd0?, 0xc000726e88?, 0xc000c059b8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0008baf60, {0xc0008bafd0?, 0xc000c05a30?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc0008baf60}, {0xc0008bafd0, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008bc5a0, {0xc0008bafd0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc0008bafc0, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc0008bafc0, 0xc0008bc5a0, {0x0, 0x0}, 0x7fffffff, 0xc000c05d38, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0xc000c05c98?, {0x7f992fe94558, 0x40f8680}, 0xc000c05ca0?, {0x0?, 0x0?}, {0x2340c20, 0xc000716e80}, 0xc000c05cb8?, 0xc000c05d38, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc0008252b0, {0x2340c20, 0xc000716e80}, 0xc000674720?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc0008bc240, 0xc000c05e58, 0xc000c05ea0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008bc240, {0x2340c20?, 0xc000716e80?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0008be220) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6853 +0x46 -go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000141ae0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:458 +0x26a -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 56 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:301 +0x58 - -goroutine 56 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() - /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 - /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 - -goroutine 22 [select]: -google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00080a0f0, 0x1) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 -google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0008558f0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 -google.golang.org/grpc/internal/transport.newHTTP2Client.func6() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 46 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 - -goroutine 44 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b0f4a0, {0x2b2cc60, 0xc000a87d10}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 45 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b0f4d0, {0x2b2cc60, 0xc000a87d60}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 82 [chan receive]: -go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea -created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 59 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c - -goroutine 24 [select]: -go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000141ae0, {0x2b2cc60, 0xc000b34b40}, 0x1f2292329c2c5814, 0xc0000c2720) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 56 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 - -goroutine 52 [select]: -google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000124248) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1694 +0x137 -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 46 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:399 +0x1e26 - -goroutine 53 [IO wait]: -internal/poll.runtime_pollWait(0x7f99312c6668, 0x72) - /usr/local/go/src/runtime/netpoll.go:345 +0x85 -internal/poll.(*pollDesc).wait(0xc0005ca080?, 0xc000ac8000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc0005ca080, {0xc000ac8000, 0x8000, 0x8000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a -net.(*netFD).Read(0xc0005ca080, {0xc000ac8000?, 0x1060100000000?, 0x8?}) - /usr/local/go/src/net/fd_posix.go:55 +0x25 -net.(*conn).Read(0xc000090510, {0xc000ac8000?, 0x800010601?, 0xc000000000?}) - /usr/local/go/src/net/net.go:185 +0x45 -bufio.(*Reader).Read(0xc0005f8ea0, {0xc000b02200, 0x9, 0x408b620?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc0005f8ea0}, {0xc000b02200, 0x9, 0x9}, 0x9) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -golang.org/x/net/http2.readFrameHeader({0xc000b02200, 0x9, 0xc0005b82e8?}, {0x2b0d3e0?, 0xc0005f8ea0?}) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 -golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b021c0) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 -google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000124248, 0xc0005f8f00) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 46 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 - -goroutine 26 [select]: -go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000141ae0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:550 +0x75 -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 56 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:302 +0x96 - -goroutine 27 [chan receive]: -go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 -created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 56 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 - -goroutine 146 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 130 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 67 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() - /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 - -goroutine 68 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() - /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 - -goroutine 69 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() - /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 66 - /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 - -goroutine 59 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() - /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 69 - /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 - -goroutine 60 [select]: -context.(*cancelCtx).propagateCancel.func2() - /usr/local/go/src/context/context.go:510 +0x98 -created by context.(*cancelCtx).propagateCancel in goroutine 59 - /usr/local/go/src/context/context.go:509 +0x3f3 - -goroutine 61 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000825380) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 -created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 59 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 - -goroutine 63 [chan receive]: -go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea -created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 80 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c - -goroutine 70 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 61 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 71 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0000c1c20, {0xc00091b450, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000c1c20, {0xc00091b450?, 0xc0005b81f8?, 0xc000a099d8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00091b3e0, {0xc00091b450?, 0xc000a09a50?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc00091b3e0}, {0xc00091b450, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008a07e0, {0xc00091b450, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc00091b440, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc00091b440, 0xc0008a07e0, {0x0, 0x0}, 0x7fffffff, 0xc000a09d58, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0xc0000ad460?, {0x0?, 0x0?}, {0x2380420, 0xc00011f030}, 0xc00023ab70?, 0xc000a09d58, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc000978270, {0x2380420, 0xc00011f030}, 0x40c573?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc0008a0480, 0xc000a09e78, 0xc000a09ec0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008a0480, {0x2380420?, 0xc00011f030?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc000902620) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000825380, {0x2b39370, 0xc000902620}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 61 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 - -goroutine 72 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000825380, 0xc000968d10, 0xc000813620) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 61 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b - -goroutine 83 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() - /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 - /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 - -goroutine 84 [select]: -go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000141ae0, {0x2b2cc60, 0xc00069f630}, 0x1f2292329c2c581d, 0xc0006b5bc0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 83 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 - -goroutine 85 [chan receive]: -go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 -created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 83 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 - -goroutine 77 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() - /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 - -goroutine 88 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New.func1() - /code/internal/pkg/service/common/servicectx/servicectx.go:100 +0x26 -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New in goroutine 15 - /code/internal/pkg/service/common/servicectx/servicectx.go:98 +0x23c - -goroutine 97 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b86930, {0x2b2cc60, 0xc000b749b0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 76 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() - /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 - /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 - -goroutine 98 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b86960, {0x2b2cc60, 0xc000b74a00}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 99 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b86990, {0x2b2cc60, 0xc000b74a50}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 109 [select]: -go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000b4d860, {0x2b2cc60, 0xc000b75360}, 0x1f2292329c2c5824, 0xc000b67620) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 76 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 - -goroutine 110 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000b75ae0, {0xc000bcd0c0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b75ae0, {0xc000bcd0c0?, 0xc000a9fb00?, 0xc000ac39b8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000bcd050, {0xc000bcd0c0?, 0xc000ac3a30?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc000bcd050}, {0xc000bcd0c0, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000bdc120, {0xc000bcd0c0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc000bcd0b0, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc000bcd0b0, 0xc000bdc120, {0x0, 0x0}, 0x7fffffff, 0xc000ac3d38, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0xc000ac3c98?, {0x7f992fe94558, 0x40f8680}, 0xc000ac3ca0?, {0x0?, 0x0?}, {0x2340c20, 0xc0006b3d80}, 0xc000ac3cb8?, 0xc000ac3d38, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc000b731e0, {0x2340c20, 0xc0006b3d80}, 0xc000c24fc0?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc000b7bd40, 0xc000ac3e58, 0xc000ac3ea0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc000b7bd40, {0x2340c20?, 0xc0006b3d80?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc000b87970) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6853 +0x46 -go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000b4d860) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:458 +0x26a -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 76 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:301 +0x58 - -goroutine 106 [select]: -google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000bba008) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1694 +0x137 -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 100 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:399 +0x1e26 - -goroutine 107 [IO wait]: -internal/poll.runtime_pollWait(0x7f99312c6570, 0x72) - /usr/local/go/src/runtime/netpoll.go:345 +0x85 -internal/poll.(*pollDesc).wait(0xc00067d980?, 0xc000baa000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc00067d980, {0xc000baa000, 0x8000, 0x8000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a -net.(*netFD).Read(0xc00067d980, {0xc000baa000?, 0x1060100000000?, 0x8?}) - /usr/local/go/src/net/fd_posix.go:55 +0x25 -net.(*conn).Read(0xc000711ae8, {0xc000baa000?, 0x800010601?, 0xc000000000?}) - /usr/local/go/src/net/net.go:185 +0x45 -bufio.(*Reader).Read(0xc000675d40, {0xc0001afee0, 0x9, 0x408b620?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc000675d40}, {0xc0001afee0, 0x9, 0x9}, 0x9) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -golang.org/x/net/http2.readFrameHeader({0xc0001afee0, 0x9, 0xc000054570?}, {0x2b0d3e0?, 0xc000675d40?}) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 -golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001afea0) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 -google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000bba008, 0xc000675da0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 100 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 - -goroutine 108 [select]: -google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000b75130, 0x1) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 -google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00011ecb0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 -google.golang.org/grpc/internal/transport.newHTTP2Client.func6() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 100 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 - -goroutine 111 [select]: -go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000b4d860) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:550 +0x75 -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 76 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:302 +0x96 - -goroutine 112 [chan receive]: -go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 -created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 76 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 - -goroutine 114 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 110 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 115 [select]: -go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000b4d860, {0x2b39318, 0xc000b87970}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:590 +0x1d2 -created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 110 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:500 +0x2ab - -goroutine 78 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() - /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 - -goroutine 79 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() - /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 - -goroutine 116 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() - /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 77 - /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 - -goroutine 80 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() - /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 116 - /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 - -goroutine 81 [select]: -context.(*cancelCtx).propagateCancel.func2() - /usr/local/go/src/context/context.go:510 +0x98 -created by context.(*cancelCtx).propagateCancel in goroutine 80 - /usr/local/go/src/context/context.go:509 +0x3f3 - -goroutine 130 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000a78f70) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 -created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 80 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 - -goroutine 132 [select]: -go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000b4d860, {0x2b2cc60, 0xc000a513b0}, 0x1f2292329c2c582d, 0xc000c38540) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 162 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 - -goroutine 147 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0009cc320, {0xc0009ca850, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0009cc320, {0xc0009ca850?, 0xc00014c138?, 0xc0004f19d8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0009ca7e0, {0xc0009ca850?, 0xc0004f1a50?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc0009ca7e0}, {0xc0009ca850, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0009e8360, {0xc0009ca850, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc0009ca840, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc0009ca840, 0xc0009e8360, {0x0, 0x0}, 0x7fffffff, 0xc0004f1d58, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0xc0000ad460?, {0x0?, 0x0?}, {0x2380420, 0xc0002a5110}, 0xc00023ab70?, 0xc0004f1d58, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc0009ea000, {0x2380420, 0xc0002a5110}, 0x40c573?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc0009e8000, 0xc0004f1e78, 0xc0004f1ec0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009e8000, {0x2380420?, 0xc0002a5110?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc0009d6300) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000a78f70, {0x2b39370, 0xc0009d6300}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 130 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 - -goroutine 148 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000a78f70, 0xc0009f6000, 0xc000c38240) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 130 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b - -goroutine 65 [IO wait]: -internal/poll.runtime_pollWait(0x7f99312c6760, 0x72) - /usr/local/go/src/runtime/netpoll.go:345 +0x85 -internal/poll.(*pollDesc).wait(0xe?, 0xc0009ce900?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Accept(0xc000c96100) - /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac -net.(*netFD).accept(0xc000c96100) - /usr/local/go/src/net/fd_unix.go:172 +0x29 -net.(*TCPListener).accept(0xc0008fa440) - /usr/local/go/src/net/tcpsock_posix.go:159 +0x1e -net.(*TCPListener).Accept(0xc0008fa440) - /usr/local/go/src/net/tcpsock.go:327 +0x30 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/tcp.(*Protocol).Accept(0xc0008c1700?, {0x2b27940?, 0xc0008fa440?}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/tcp/protocol_tcp.go:26 +0x22 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnection(0xc0008c1700, {0x2b2cc28, 0xc000c8c570}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:218 +0x62 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnectionsLoop(0xc0008c1700, {0x2b2cc28, 0xc000c8c570}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:204 +0x67 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.Listen.func1() - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:68 +0x5a -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.Listen in goroutine 15 - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:66 +0x548 - -goroutine 162 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() - /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 - /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 - -goroutine 133 [chan receive]: -go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 -created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 162 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 - -goroutine 165 [semacquire]: -sync.runtime_Semacquire(0x44d74b?) - /usr/local/go/src/runtime/sema.go:62 +0x25 -sync.(*WaitGroup).Wait(0x1cfe4fc?) - /usr/local/go/src/sync/waitgroup.go:116 +0x48 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close(0x2b5c500, {0x2b2cc28, 0xc000d0a6c0}, {0x2401575, 0x8}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:129 +0x22b -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router.New.func1({0x2b2cc28, 0xc000d0a6c0}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/router.go:92 +0x158 -github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New.func1() - /code/internal/pkg/service/common/servicectx/servicectx.go:104 +0x51 -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.New in goroutine 15 - /code/internal/pkg/service/common/servicectx/servicectx.go:98 +0x23c - -goroutine 149 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).Accept(0x2b423c0?) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:76 +0x67 -google.golang.org/grpc.(*Server).Serve(0xc000789600, {0x2b2aae0, 0xc0008c1700}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:885 +0x49e -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.(*NetworkFileServer).serve(0xc0008fcaa0, {0x2b2aae0, 0xc0008c1700}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/server.go:121 +0x2a5 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.StartNetworkFileServer.func2(0xc0005a3390) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/server.go:104 +0x2a -github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.(*Process).Add.func1() - /code/internal/pkg/service/common/servicectx/servicectx.go:174 +0x9d -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx.(*Process).Add in goroutine 15 - /code/internal/pkg/service/common/servicectx/servicectx.go:172 +0x77 - -goroutine 150 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/node/writernode/diskcleanup.Start.func2() - /code/internal/pkg/service/stream/storage/node/writernode/diskcleanup/diskcleanup.go:100 +0x1ff -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/node/writernode/diskcleanup.Start in goroutine 15 - /code/internal/pkg/service/stream/storage/node/writernode/diskcleanup/diskcleanup.go:89 +0x319 - -goroutine 125 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d7250, {0x2b2cc60, 0xc000be0a50}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 212 [select]: -github.com/hashicorp/yamux.(*Session).keepalive(0xc0009e29c0) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:349 +0x85 -created by github.com/hashicorp/yamux.newSession in goroutine 30 - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:121 +0x4cf - -goroutine 213 [select]: -github.com/hashicorp/yamux.(*Session).AcceptStream(0xc000befe00) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:242 +0x69 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptStream(0xc0008c1700, 0xc000befe00) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:271 +0x2a -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptStreamsLoop(0xc0008c1700, {0x2b2cc28, 0xc000c8c570}, 0xc000befe00) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:257 +0x86 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnection.func1() - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:242 +0xa5 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*Server).acceptConnection in goroutine 65 - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/server.go:240 +0x39b - -goroutine 211 [select]: -github.com/hashicorp/yamux.(*Session).sendLoop(0xc0009e29c0) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:467 +0x11d -github.com/hashicorp/yamux.(*Session).send(0xc0009e29c0) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:456 +0x18 -created by github.com/hashicorp/yamux.newSession in goroutine 30 - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:119 +0x485 - -goroutine 183 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() - /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 161 - /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 - -goroutine 126 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d7280, {0x2b2cc60, 0xc000be0aa0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 158 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000a50640, {0xc000d8d150, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a50640, {0xc000d8d150?, 0xc0006ae468?, 0xc0011099b8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000d8d0e0, {0xc000d8d150?, 0xc001109a30?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc000d8d0e0}, {0xc000d8d150, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008266c0, {0xc000d8d150, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc000d8d140, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc000d8d140, 0xc0008266c0, {0x0, 0x0}, 0x7fffffff, 0xc001109d38, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0xc001109c98?, {0x7f992fe94558, 0x40f8680}, 0xc001109ca0?, {0x0?, 0x0?}, {0x2340c20, 0xc0006b3bc0}, 0xc001109cb8?, 0xc001109d38, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc000a781a0, {0x2340c20, 0xc0006b3bc0}, 0xc0005f9f20?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc000826360, 0xc001109e58, 0xc001109ea0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc000826360, {0x2340c20?, 0xc0006b3bc0?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc000c20680) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6853 +0x46 -go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000e96c80) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:458 +0x26a -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 156 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:301 +0x58 - -goroutine 127 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d72b0, {0x2b2cc60, 0xc000be0af0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 30 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.(*ClientConnection).dialLoop(0xc0002a41c0, {0x2b2cc28, 0xc000bcc540}, 0x0) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/client_conn.go:195 +0x67c -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.newClientConnection.func1() - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/client_conn.go:58 +0x73 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport.newClientConnection in goroutine 29 - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/transport/client_conn.go:56 +0x5ab - -goroutine 156 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start.func1() - /code/internal/pkg/service/common/etcdop/session.go:181 +0x4ac -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.SessionBuilder.Start in goroutine 15 - /code/internal/pkg/service/common/etcdop/session.go:127 +0x345 - -goroutine 157 [select]: -go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000e96c80, {0x2b2cc60, 0xc0008fd810}, 0x1f2292329c2c5836, 0xc0000c33e0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:332 +0xcb -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 156 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:298 +0x567 - -goroutine 153 [select]: -google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000be11d0, 0x1) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 -google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000f08c0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 -google.golang.org/grpc/internal/transport.newHTTP2Client.func6() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 128 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 - -goroutine 181 [select]: -google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000688008) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1694 +0x137 -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 128 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:399 +0x1e26 - -goroutine 182 [IO wait]: -internal/poll.runtime_pollWait(0x7f99312c6478, 0x72) - /usr/local/go/src/runtime/netpoll.go:345 +0x85 -internal/poll.(*pollDesc).wait(0xc000682580?, 0xc000316000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000682580, {0xc000316000, 0x8000, 0x8000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a -net.(*netFD).Read(0xc000682580, {0xc000316000?, 0x1060100000000?, 0x8?}) - /usr/local/go/src/net/fd_posix.go:55 +0x25 -net.(*conn).Read(0xc000710e18, {0xc000316000?, 0x800010601?, 0xc000000000?}) - /usr/local/go/src/net/net.go:185 +0x45 -bufio.(*Reader).Read(0xc000bc9620, {0xc0001af380, 0x9, 0xc000093008?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc000bc9620}, {0xc0001af380, 0x9, 0x9}, 0x9) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -golang.org/x/net/http2.readFrameHeader({0xc0001af380, 0x9, 0xc0006ae480?}, {0x2b0d3e0?, 0xc000bc9620?}) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 -golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001af340) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 -google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000688008, 0xc000bc9680) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 128 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 - -goroutine 159 [select]: -go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000e96c80) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:550 +0x75 -created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 156 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:302 +0x96 - -goroutine 160 [chan receive]: -go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:66 +0x74 -created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 156 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/concurrency/session.go:64 +0x256 - -goroutine 167 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 158 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 168 [select]: -go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000e96c80, {0x2b39318, 0xc000c20680}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:590 +0x1d2 -created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 158 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/lease.go:500 +0x2ab - -goroutine 134 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() - /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 - -goroutine 135 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() - /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 - -goroutine 136 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() - /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 - -goroutine 161 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() - /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 134 - /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 - -goroutine 184 [select]: -context.(*cancelCtx).propagateCancel.func2() - /usr/local/go/src/context/context.go:510 +0x98 -created by context.(*cancelCtx).propagateCancel in goroutine 183 - /usr/local/go/src/context/context.go:509 +0x3f3 - -goroutine 185 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc0009eb450) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 -created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 183 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 - -goroutine 210 [IO wait]: -internal/poll.runtime_pollWait(0x7f99312c6380, 0x72) - /usr/local/go/src/runtime/netpoll.go:345 +0x85 -internal/poll.(*pollDesc).wait(0xc0009e6780?, 0xc000268000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc0009e6780, {0xc000268000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a -net.(*netFD).Read(0xc0009e6780, {0xc000268000?, 0x2b0d3e0?, 0xc0009db620?}) - /usr/local/go/src/net/fd_posix.go:55 +0x25 -net.(*conn).Read(0xc0009e4070, {0xc000268000?, 0xc000093008?, 0xc000d00720?}) - /usr/local/go/src/net/net.go:185 +0x45 -bufio.(*Reader).Read(0xc0009db620, {0xc0005b4f70, 0xc, 0x0?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc0009db620}, {0xc0005b4f70, 0xc, 0xc}, 0xc) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/hashicorp/yamux.(*Session).recvLoop(0xc0009e29c0) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:536 +0xb9 -github.com/hashicorp/yamux.(*Session).recv(0xc0009e29c0) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:515 +0x18 -created by github.com/hashicorp/yamux.newSession in goroutine 30 - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:118 +0x446 - -goroutine 187 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 185 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 188 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000be1a90, {0xc000c8dd80, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000be1a90, {0xc000c8dd80?, 0xc000a9ec78?, 0xc000bf79d8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c8dd10, {0xc000c8dd80?, 0xc000bf7a50?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc000c8dd10}, {0xc000c8dd80, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0004d3560, {0xc000c8dd80, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc000c8dd70, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc000c8dd70, 0xc0004d3560, {0x0, 0x0}, 0x7fffffff, 0xc000bf7d58, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0x0?, {0x0?, 0x0?}, {0x2380420, 0xc00011f0a0}, 0xc000bc9620?, 0xc000bf7d58, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc0009eb6c0, {0x2380420, 0xc00011f0a0}, 0x40c573?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc0009e9d40, 0xc000bf7e78, 0xc000bf7ec0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009e9d40, {0x2380420?, 0xc00011f0a0?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc000850260) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc0009eb450, {0x2b39370, 0xc000850260}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 185 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 - -goroutine 189 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0009eb450, 0xc0009f62c0, 0xc000be9620) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 185 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b - -goroutine 194 [chan receive]: -go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea -created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 183 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c - -goroutine 195 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() - /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 - -goroutine 196 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() - /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x24d -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 - -goroutine 197 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() - /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 - -goroutine 169 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() - /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 195 - /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 - -goroutine 170 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() - /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 169 - /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 - -goroutine 171 [select]: -context.(*cancelCtx).propagateCancel.func2() - /usr/local/go/src/context/context.go:510 +0x98 -created by context.(*cancelCtx).propagateCancel in goroutine 170 - /usr/local/go/src/context/context.go:509 +0x3f3 - -goroutine 172 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000a78410) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 -created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 170 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 - -goroutine 174 [chan receive]: -go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea -created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 235 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c - -goroutine 198 [IO wait]: -internal/poll.runtime_pollWait(0x7f99312c6288, 0x72) - /usr/local/go/src/runtime/netpoll.go:345 +0x85 -internal/poll.(*pollDesc).wait(0xc000c47900?, 0xc000b4b000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000c47900, {0xc000b4b000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a -net.(*netFD).Read(0xc000c47900, {0xc000b4b000?, 0x2b0d3e0?, 0xc0006740c0?}) - /usr/local/go/src/net/fd_posix.go:55 +0x25 -net.(*conn).Read(0xc0005ce550, {0xc000b4b000?, 0x40cab0?, 0xc000c84e60?}) - /usr/local/go/src/net/net.go:185 +0x45 -bufio.(*Reader).Read(0xc0006740c0, {0xc0009ddcb0, 0xc, 0x0?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc0006740c0}, {0xc0009ddcb0, 0xc, 0xc}, 0xc) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/hashicorp/yamux.(*Session).recvLoop(0xc000befe00) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:536 +0xb9 -github.com/hashicorp/yamux.(*Session).recv(0xc000befe00) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:515 +0x18 -created by github.com/hashicorp/yamux.newSession in goroutine 65 - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:118 +0x446 - -goroutine 199 [select]: -github.com/hashicorp/yamux.(*Session).sendLoop(0xc000befe00) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:467 +0x11d -github.com/hashicorp/yamux.(*Session).send(0xc000befe00) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:456 +0x18 -created by github.com/hashicorp/yamux.newSession in goroutine 65 - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:119 +0x485 - -goroutine 200 [select]: -github.com/hashicorp/yamux.(*Session).keepalive(0xc000befe00) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:349 +0x85 -created by github.com/hashicorp/yamux.newSession in goroutine 65 - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/session.go:121 +0x4cf - -goroutine 226 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 172 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 227 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000a86550, {0xc0007ac2e0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a86550, {0xc0007ac2e0?, 0xc000054318?, 0xc0007539d8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0007ac270, {0xc0007ac2e0?, 0xc000753a50?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc0007ac270}, {0xc0007ac2e0, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000afe360, {0xc0007ac2e0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc0007ac2d0, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc0007ac2d0, 0xc000afe360, {0x0, 0x0}, 0x7fffffff, 0xc000753d58, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0x0?, {0x0?, 0x0?}, {0x2380420, 0xc00029c070}, 0x0?, 0xc000753d58, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc000aaa1a0, {0x2380420, 0xc00029c070}, 0x40c573?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc000afe000, 0xc000753e78, 0xc000753ec0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc000afe000, {0x2380420?, 0xc00029c070?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc00088e9b0) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000a78410, {0x2b39370, 0xc00088e9b0}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 172 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 - -goroutine 228 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000a78410, 0xc00085e2c0, 0xc000c38fc0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 172 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b - -goroutine 229 [chan receive]: -go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:59 +0xea -created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 170 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/namespace/watch.go:54 +0x35c - -goroutine 230 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/statistics/collector.Start.func4() - /code/internal/pkg/service/stream/storage/statistics/collector/collector.go:128 +0x115 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/statistics/collector.Start in goroutine 15 - /code/internal/pkg/service/stream/storage/statistics/collector/collector.go:122 +0x445 - -goroutine 231 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart.func1() - /code/internal/pkg/service/common/etcdop/watch_restart.go:77 +0x49c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.wrapStreamWithRestart in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_restart.go:50 +0x1d1 - -goroutine 232 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel.func1() - /code/internal/pkg/service/common/etcdop/watch_typed.go:88 +0x253 -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].decodeChannel in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_typed.go:70 +0x1f5 - -goroutine 233 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer.func1() - /code/internal/pkg/service/common/etcdop/watch_consumer.go:125 +0x14c -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.(*WatchConsumer[...]).StartConsumer in goroutine 15 - /code/internal/pkg/service/common/etcdop/watch_consumer.go:111 +0x2b4 - -goroutine 234 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1.1() - /code/internal/pkg/service/common/etcdop/watch.go:116 +0x50b -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.PrefixT[...].GetAllAndWatch.Prefix.GetAllAndWatch.func1 in goroutine 231 - /code/internal/pkg/service/common/etcdop/watch.go:76 +0x1b3 - -goroutine 235 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart.func1() - /code/internal/pkg/service/common/etcdop/watch.go:161 +0x2ca -created by github.com/keboola/keboola-as-code/internal/pkg/service/common/etcdop.Prefix.WatchWithoutRestart in goroutine 234 - /code/internal/pkg/service/common/etcdop/watch.go:144 +0x1f3 - -goroutine 236 [select]: -context.(*cancelCtx).propagateCancel.func2() - /usr/local/go/src/context/context.go:510 +0x98 -created by context.(*cancelCtx).propagateCancel in goroutine 235 - /usr/local/go/src/context/context.go:509 +0x3f3 - -goroutine 237 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000aaa4e0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:545 +0x337 -created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 235 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:292 +0x2e5 - -goroutine 239 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 237 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 240 [select]: -google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000a87680, {0xc0007adcf0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:193 +0x95 -google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a87680, {0xc0007adcf0?, 0xc000054648?, 0xc0008759d8?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:173 +0x12d -google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0007adc80, {0xc0007adcf0?, 0xc000875a50?, 0xbeb985?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:525 +0x2c -io.ReadAtLeast({0x2b11be0, 0xc0007adc80}, {0xc0007adcf0, 0x5, 0x5}, 0x5) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000aff9e0, {0xc0007adcf0, 0x5, 0x5}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:509 +0x96 -google.golang.org/grpc.(*parser).recvMsg(0xc0007adce0, 0x7fffffff) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:614 +0x46 -google.golang.org/grpc.recvAndDecompress(0xc0007adce0, 0xc000aff9e0, {0x0, 0x0}, 0x7fffffff, 0xc000875d58, {0x0, 0x0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:753 +0x85 -google.golang.org/grpc.recv(0x2b1c660?, {0x7f992fe94558, 0x40f8680}, 0x0?, {0x0?, 0x0?}, {0x2380420, 0xc00029c310}, 0x0?, 0xc000875d58, ...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/rpc_util.go:833 +0x7d -google.golang.org/grpc.(*csAttempt).recvMsg(0xc000aaac30, {0x2380420, 0xc00029c310}, 0x40c573?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1086 +0x289 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x70?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc000afed80, 0xc000875e78, 0xc000875ec0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:762 +0x3ae -google.golang.org/grpc.(*clientStream).RecvMsg(0xc000afed80, {0x2380420?, 0xc00029c310?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc00088f590) - /tmp/cache/go-mod/go.etcd.io/etcd/api/v3@v3.5.14/etcdserverpb/rpc.pb.go:6714 +0x46 -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000aaa4e0, {0x2b39370, 0xc00088f590}) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:766 +0x5c -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 237 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:921 +0x585 - -goroutine 241 [select]: -go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000aaa4e0, 0xc00085ea50, 0xc0006b59e0) - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:812 +0x23e -created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 237 - /tmp/cache/go-mod/go.etcd.io/etcd/client/v3@v3.5.14/watch.go:562 +0xc1b - -goroutine 279 [sync.RWMutex.Lock]: -sync.runtime_SemacquireRWMutex(0x1?, 0x1?, 0x1ed9120?) - /usr/local/go/src/runtime/sema.go:87 +0x25 -sync.(*RWMutex).Lock(0xc0010f0280?) - /usr/local/go/src/sync/rwmutex.go:151 +0x6a -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SlicePipeline).Close(0xc001060630, {0x2b2cc60, 0xc00080ab90}, {0x2401575, 0x8}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_slice.go:129 +0x65 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close.func1() - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:126 +0x71 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close in goroutine 278 - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:124 +0x15c - -goroutine 278 [semacquire]: -sync.runtime_Semacquire(0x44d74b?) - /usr/local/go/src/runtime/sema.go:62 +0x25 -sync.(*WaitGroup).Wait(0x1cfa95c?) - /usr/local/go/src/sync/waitgroup.go:116 +0x48 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close(0x2b5c680, {0x2b2ccd0, 0xc0002a6a80}, {0x2401575, 0x8}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:129 +0x22b -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SinkPipeline).close(0xc00099fe00, {0x2b2ccd0, 0xc0002a6a80}, {0x2401575, 0x8}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_sink.go:190 +0x114 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SinkPipeline).Close(0xc00099fe00, {0x2b2ccd0, 0xc0002a6a80}, {0x2401575, 0x8}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_sink.go:177 +0xd4 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close.func1() - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:126 +0x71 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*Collection[...]).Close in goroutine 165 - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/collection.go:124 +0x15c - -goroutine 266 [semacquire]: -sync.runtime_Semacquire(0xc000c2bd80?) - /usr/local/go/src/runtime/sema.go:62 +0x25 -sync.(*WaitGroup).Wait(0xc000e80ea0?) - /usr/local/go/src/sync/waitgroup.go:116 +0x48 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).DispatchToSource(0xc000ab0360, {{0x2b0de40?, 0xc000aa8ed0?}, {0x2402fcb?, 0x61?}}, {0x2b3a1e0, 0xc0006ba420}) - /code/internal/pkg/service/stream/sink/router/router.go:255 +0x2b7 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testcase.(*WriterTestCase).Run.func4() - /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:136 +0xc5 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testcase.(*WriterTestCase).Run in goroutine 15 - /code/internal/pkg/service/stream/storage/test/testcase/testcase.go:133 +0x1130 - -goroutine 267 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync/notify.(*Notifier).Wait(0xc00113a4b0, {0x2b2c7e0, 0x40f8680}) - /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/notify/notify.go:37 +0x7e -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.(*pipeline).WriteRecord(0xc0010f4c60, {0x2b3a1e0, 0xc0006ba420}) - /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:289 +0x24c -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SlicePipeline).WriteRecord(0xc001060630, {0x2b3a1e0, 0xc0006ba420}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_slice.go:114 +0xe8 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/balancer.RoundRobinBalancer.WriteRecord({0x1?}, {0x2b3a1e0?, 0xc0006ba420?}, {0xc0011389c0?, 0x6f?, 0xc000e403c0?}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/balancer/balancer_rr.go:28 +0x77 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline.(*SinkPipeline).WriteRecord(0xc000704a90?, {0x2b3a1e0?, 0xc0006ba420?}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/router/pipeline/pipeline_sink.go:98 +0xe3 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*pipelineRef).writeRecord(0xc000704a90, {0x2b3a1e0, 0xc0006ba420}) - /code/internal/pkg/service/stream/sink/router/pipeline.go:55 +0x62 -github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).writeRecord(0xc000ab0360, 0xc000a877c0, {0x2b3a1e0, 0xc0006ba420}) - /code/internal/pkg/service/stream/sink/router/router.go:314 +0x6e -github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).dispatchToSink(0xc000ab0360, 0xc000a877c0, {0x2b3a1e0, 0xc0006ba420}) - /code/internal/pkg/service/stream/sink/router/router.go:283 +0x8b -github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).DispatchToSource.func1() - /code/internal/pkg/service/stream/sink/router/router.go:234 +0xb4 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/router.(*Router).DispatchToSource in goroutine 266 - /code/internal/pkg/service/stream/sink/router/router.go:230 +0x2a5 - -goroutine 269 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010ab6f0, {0x2b2cc60, 0xc0010a9c20}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 268 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 270 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010ab720, {0x2b2cc60, 0xc0010a9cc0}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 268 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 271 [select]: -google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010ab750, {0x2b2cc60, 0xc0010a9d10}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x115 -created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 268 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x11a - -goroutine 277 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.(*NetworkFileServer).KeepAliveStream(0xc0008fcaa0, 0xc0006578c0, {0x2b38160, 0xc000ec8130}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/server.go:157 +0xaa -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/pb._NetworkFile_KeepAliveStream_Handler({0x22d8ca0, 0xc0008fcaa0}, {0x2b33e40, 0xc0010b4000}) - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/pb/networkFile_grpc.pb.go:185 +0x107 -google.golang.org/grpc.(*Server).processStreamingRPC(0xc000789600, {0x2b2cc28, 0xc0006576e0}, {0x2b3a360, 0xc000b5c780}, 0xc0001a4120, 0xc0007a7560, 0x3dfaee0, 0x0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1673 +0x1208 -google.golang.org/grpc.(*Server).handleStream(0xc000789600, {0x2b3a360, 0xc000b5c780}, 0xc0001a4120) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1794 +0xe3a -google.golang.org/grpc.(*Server).serveStreams.func2.1() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1029 +0x8b -created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 207 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1040 +0x125 - -goroutine 276 [select]: -google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001102320, 0x1) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 -google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00011f420) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 -google.golang.org/grpc/internal/transport.newHTTP2Client.func6() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0xbb -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 272 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x2492 - -goroutine 275 [select]: -github.com/hashicorp/yamux.(*Stream).Read(0xc0010fe5b0, {0xc001112000, 0x8000, 0xc000c85d30?}) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/stream.go:145 +0x40e -bufio.(*Reader).Read(0xc000c272c0, {0xc000f4d7e0, 0x9, 0x408b620?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc000c272c0}, {0xc000f4d7e0, 0x9, 0x9}, 0x9) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -golang.org/x/net/http2.readFrameHeader({0xc000f4d7e0, 0x9, 0x21290a0?}, {0x2b0d3e0?, 0xc000c272c0?}) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 -golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f4d7a0) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 -google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000689448, 0xc000c27320) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x22d -created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 272 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x1e99 - -goroutine 205 [select]: -google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00080b360, 0x1) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x113 -google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00011f3b0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x86 -google.golang.org/grpc/internal/transport.NewServerTransport.func2() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:334 +0xc7 -created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 204 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:332 +0x193e - -goroutine 206 [select]: -google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b5c780) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:1168 +0x205 -created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 204 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:355 +0x1985 - -goroutine 207 [select]: -github.com/hashicorp/yamux.(*Stream).Read(0xc0008251e0, {0xc000eb8000, 0x8000, 0x0?}) - /tmp/cache/go-mod/github.com/hashicorp/yamux@v0.1.1/stream.go:145 +0x40e -bufio.(*Reader).Read(0xc000675c80, {0xc000a1e2e0, 0x9, 0x408b620?}) - /usr/local/go/src/bufio/bufio.go:241 +0x197 -io.ReadAtLeast({0x2b0d3e0, 0xc000675c80}, {0xc000a1e2e0, 0x9, 0x9}, 0x9) - /usr/local/go/src/io/io.go:335 +0x90 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -golang.org/x/net/http2.readFrameHeader({0xc000a1e2e0, 0x9, 0xc0005b8120?}, {0x2b0d3e0?, 0xc000675c80?}) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:237 +0x65 -golang.org/x/net/http2.(*Framer).ReadFrame(0xc000a1e2a0) - /tmp/cache/go-mod/golang.org/x/net@v0.26.0/http2/frame.go:501 +0x85 -google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b5c780, {0x2b2cc28, 0xc000c146f0}, 0xc000c14780) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_server.go:644 +0x15e -google.golang.org/grpc.(*Server).serveStreams(0xc000789600, {0x2b2c7e0?, 0x40f8680?}, {0x2b3a360, 0xc000b5c780}, {0x2b39840?, 0xc000da3c60?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:1023 +0x3b6 -google.golang.org/grpc.(*Server).handleRawConn.func1() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:959 +0x56 -created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 204 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/server.go:958 +0x1c6 - -goroutine 137 [select]: -google.golang.org/grpc.newClientStreamWithParams.func4() - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:392 +0x8c -created by google.golang.org/grpc.newClientStreamWithParams in goroutine 268 - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:391 +0xe08 - -goroutine 138 [select]: -google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00103a240) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:330 +0x7c -google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/internal/transport/transport.go:345 -google.golang.org/grpc.(*csAttempt).recvMsg(0xc0010ff5f0, {0x21553c0, 0xc000c14030}, 0x0?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:1072 +0xc9 -google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x30?) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:929 +0x1f -google.golang.org/grpc.(*clientStream).withRetry(0xc00103a000, 0xc000eb6e80, 0xc000eb6ec8) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:778 +0x13a -google.golang.org/grpc.(*clientStream).RecvMsg(0xc00103a000, {0x21553c0?, 0xc000c14030?}) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream.go:928 +0x110 -google.golang.org/grpc.(*GenericClientStream[...]).Recv(0x2b3b0c0) - /tmp/cache/go-mod/google.golang.org/grpc@v1.64.0/stream_interfaces.go:99 +0x4e -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.OpenNetworkFile.func2() - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/client.go:124 +0x62 -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc.OpenNetworkFile in goroutine 268 - /code/internal/pkg/service/stream/storage/level/local/diskwriter/network/rpc/client.go:122 +0x925 - -goroutine 139 [chan receive]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.(*pipeline).processChunks(0xc0010f4c60, {0x2b2cc28, 0xc0010fdaa0}, {0x2b3b3f0, 0x40f8680}, {{{0xc00013b79c, 0x3}, 0x0, 0x180000, {0x0, ...}}, ...}) - /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:418 +0x6c -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.newPipeline.func2() - /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:165 +0xbe -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.newPipeline in goroutine 268 - /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:163 +0x72e - -goroutine 140 [select]: -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).syncLoop.func1() - /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:242 +0xea -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).syncLoop in goroutine 268 - /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:236 +0x98 - -goroutine 177 [sync.RWMutex.Lock]: -sync.runtime_SemacquireRWMutex(0xc000eb7de8?, 0xff?, 0xc1b5ba829e8d0880?) - /usr/local/go/src/runtime/sema.go:87 +0x25 -sync.(*RWMutex).Lock(0x0?) - /usr/local/go/src/sync/rwmutex.go:151 +0x6a -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding.(*pipeline).Flush(0xc0010f4c60, {0x2b2c7e0, 0x40f8680}) - /code/internal/pkg/service/stream/storage/level/local/encoding/pipeline.go:339 +0x5c -github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).TriggerSync.func1() - /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:184 +0x17e -created by github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/writesync.(*Syncer).TriggerSync in goroutine 140 - /code/internal/pkg/service/stream/storage/level/local/encoding/writesync/sync.go:176 +0x225 -FAIL github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/encoding/encoder/csv 10.062s -FAIL