From f6373e4c2ead531811f2e502d0ce099a26fadf1a Mon Sep 17 00:00:00 2001
From: Tim <timdortmann@gmail.com>
Date: Sun, 24 Jul 2022 04:13:57 +0200
Subject: [PATCH] limit goroutines to cpu count

---
 .gitignore |  2 +-
 go.mod     |  1 +
 go.sum     |  2 ++
 main.go    | 18 ++++++++++++------
 4 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2cff6c9..5583008 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,7 +10,7 @@
 
 # Test binary, built with `go test -c`
 *.test
-output/
+/output
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
diff --git a/go.mod b/go.mod
index 9c9e204..126ccec 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@ require (
 )
 
 require (
+	github.com/alitto/pond v1.8.0
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
diff --git a/go.sum b/go.sum
index 5b2732f..601aba4 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,7 @@
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/alitto/pond v1.8.0 h1:/4wnAU0vOjhsUxOxjtXuNb59oh0J+Jjukf6gtkWpGJk=
+github.com/alitto/pond v1.8.0/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
diff --git a/main.go b/main.go
index f2dbdf0..8842f45 100644
--- a/main.go
+++ b/main.go
@@ -3,17 +3,19 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/PuerkitoBio/goquery"
-	"github.com/schollz/progressbar/v3"
-	"go.uber.org/ratelimit"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"sync"
 	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/schollz/progressbar/v3"
+	"go.uber.org/ratelimit"
 )
 
 type inputOptions struct {
@@ -99,7 +101,7 @@ func main() {
 	}
 
 	dl_bar := progressbar.NewOptions(len(posts),
-		progressbar.OptionSetDescription(fmt.Sprintf("Downloading posts")),
+		progressbar.OptionSetDescription("Downloading posts"),
 		progressbar.OptionEnableColorCodes(true),
 		progressbar.OptionFullWidth(),
 		progressbar.OptionShowCount(),
@@ -116,11 +118,16 @@ func main() {
 	wg.Add(len(posts))
 	start := time.Now()
 
+	maxGoroutines := runtime.NumCPU()
+	guard := make(chan Post, maxGoroutines)
+
 	for _, post := range posts {
+		guard <- post
 		go func(post Post) {
 			defer wg.Done()
 			downloadPost(post, options)
 			dl_bar.Add(1)
+			<-guard
 		}(post)
 	}
 	wg.Wait()
@@ -174,7 +181,6 @@ func downloadPost(post Post, options inputOptions) {
 		fmt.Println("Error writing post:", post.ID)
 		return
 	}
-
 }
 
 func fetchPostsFromPage(tag string, totalPageAmount int) []Post {
@@ -185,7 +191,7 @@ func fetchPostsFromPage(tag string, totalPageAmount int) []Post {
 	rl := ratelimit.New(10)
 
 	pages_bar := progressbar.NewOptions(totalPageAmount,
-		progressbar.OptionSetDescription(fmt.Sprintf("Fetching posts")),
+		progressbar.OptionSetDescription("Fetching posts"),
 		progressbar.OptionEnableColorCodes(true),
 		progressbar.OptionFullWidth(),
 		progressbar.OptionShowCount(),