Parallelism added to colly

This is a fix for the "Too many requests" error
luckylittle · Jul 11, 2020 · 2a37bd5 · 2a37bd5
1 parent 404559d
commit 2a37bd5
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/download/download.go b/download/download.go
@@ -31,7 +31,8 @@ func main() {
 	for scanner.Scan() {
 
 		// instantiate default collector
-		c := colly.NewCollector()
+		c := colly.NewCollector(colly.MaxDepth(2), colly.Async(true))
+		c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 2})
 
 		// get the authenticity token
 		c.OnHTML("div.loginV2--login div[class=loginV2__form] input[type=hidden][name=authenticity_token]", func(e *colly.HTMLElement) {