diff --git a/go.mod b/go.mod index f991dffe..5bc5eac4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.16 require ( github.com/arduino/arduino-cli v0.0.0-20210520100059-2666b6ec51e9 github.com/arduino/go-paths-helper v1.6.1 - github.com/arduino/golang-concurrent-workers v0.0.0-20170202182617-6710cdc954bc github.com/go-git/go-git/v5 v5.4.2 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 52fb4123..74e438f0 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4l github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b/go.mod h1:uwGy5PpN4lqW97FiLnbcx+xx8jly5YuPMJWfVwwjJiQ= github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b h1:3PjgYG5gVPA7cipp7vIR2lF96KkEJIFBJ+ANnuv6J20= github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b/go.mod h1:iIPnclBMYm1g32Q5kXoqng4jLhMStReIP7ZxaoUC2y8= -github.com/arduino/golang-concurrent-workers v0.0.0-20170202182617-6710cdc954bc h1:PzGY1Ppud/Ng+LFHU16oOrWhYsnSLYurwiHlbVc/FJ0= -github.com/arduino/golang-concurrent-workers v0.0.0-20170202182617-6710cdc954bc/go.mod h1:E+WBbLkFBdPp+N+yijgbdDI33mr5pm6j42RYLN5K4do= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= diff --git a/internal/command/sync/sync.go b/internal/command/sync/sync.go index e190bb96..d256ef39 100644 --- a/internal/command/sync/sync.go +++ b/internal/command/sync/sync.go @@ -32,8 +32,9 @@ import ( "log" "os" "path/filepath" + "runtime" + "sync" - cc "github.com/arduino/golang-concurrent-workers" "github.com/arduino/libraries-repository-engine/internal/configuration" "github.com/arduino/libraries-repository-engine/internal/feedback" "github.com/arduino/libraries-repository-engine/internal/libraries" @@ -79,54 +80,40 @@ func syncLibraries(reposFile string) { os.Exit(1) } - type jobContext struct { - id int - repoMetadata *libraries.Repo - } - libraryDb := db.Init(config.LibrariesDB) - jobQueue := make(chan *jobContext) - - pool := cc.New(4) - worker := func() { - log.Println("Started worker...") - for job := range jobQueue { - buffer := &bytes.Buffer{} - logger := log.New(buffer, "", log.LstdFlags|log.LUTC) - syncLibrary(logger, job.repoMetadata, libraryDb) - - // Output log to file - if err := outputLogFile(logger, job.repoMetadata, buffer); err != nil { - logger.Printf("Error writing log file: %s", err.Error()) - } - - // Output log to stdout - fmt.Println(buffer.String()) - } - log.Println("Completed worker!") - } - pool.Run(worker) - pool.Run(worker) - pool.Run(worker) - pool.Run(worker) - pool.Wait() - + reposChan := make(chan *libraries.Repo) go func() { - id := 0 for _, repo := range repos { - jobQueue <- &jobContext{ - id: id, - repoMetadata: repo, - } - id++ + reposChan <- repo } - close(jobQueue) + close(reposChan) }() - for err := range pool.Errors { - feedback.LogError(err) + // Run workers in parallel to consume repositories list + var wg sync.WaitGroup + for workersCount := 0; workersCount < runtime.NumCPU(); workersCount++ { + wg.Add(1) + go func() { + log.Println("Started worker...") + for repo := range reposChan { + buffer := &bytes.Buffer{} + logger := log.New(buffer, "", log.LstdFlags|log.LUTC) + syncLibrary(logger, repo, libraryDb) + + // Output log to file + if err := outputLogFile(logger, repo, buffer); err != nil { + logger.Printf("Error writing log file: %s", err.Error()) + } + + // Output log to stdout + fmt.Println(buffer.String()) + } + wg.Done() + log.Println("Completed worker!") + }() } + wg.Wait() libraryIndex, err := libraryDb.OutputLibraryIndex() if feedback.LogError(err) {