refactored main + writers

main.go

@@ -16,8 +16,7 @@ import (
 
 var version = "dev"
 
-func runScraper(s scraper.Scraper, itemsChannel chan map[string]interface{}, globalConfig *scraper.GlobalConfig, wg *sync.WaitGroup) {
-	defer wg.Done()
+func runScraper(s *scraper.Scraper, itemsChannel chan map[string]interface{}, globalConfig *scraper.GlobalConfig) {
 	log.Printf("scraping %s\n", s.Name)
 	// This could probably be improved. We could pass the channel to
 	// GetItems instead of waiting for the scraper to finish.
@@ -134,11 +133,17 @@ func main() {
 	for _, s := range config.Scrapers {
 		if *singleScraper == "" || *singleScraper == s.Name {
 			scraperWg.Add(1)
-			go runScraper(s, itemsChannel, &config.Global, &scraperWg)
+			go func(scr scraper.Scraper) {
+				defer scraperWg.Done()
+				runScraper(&scr, itemsChannel, &config.Global)
+			}(s)
 		}
 	}
 	writerWg.Add(1)
-	go writer.Write(itemsChannel, &writerWg)
+	go func() {
+		defer writerWg.Done()
+		writer.Write(itemsChannel)
+	}()
 	scraperWg.Wait()
 	close(itemsChannel)
 	writerWg.Wait()

output/api.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"net/http"
 	"net/url"
-	"sync"
 	"time"
 )
 
@@ -25,8 +24,7 @@ func NewAPIWriter(wc *WriterConfig) *APIWriter {
 	}
 }
 
-func (f *APIWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup) {
-	defer wg.Done()
+func (f *APIWriter) Write(items chan map[string]interface{}) {
 	client := &http.Client{
 		Timeout: time.Second * 10,
 	}

output/file.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"log"
 	"os"
-	"sync"
 )
 
 type FileWriter struct {
@@ -18,8 +17,7 @@ func NewFileWriter(wc *WriterConfig) *FileWriter {
 		writerConfig: wc,
 	}
 }
-func (fr *FileWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup) {
-	defer wg.Done()
+func (fr *FileWriter) Write(items chan map[string]interface{}) {
 	f, err := os.Create(fr.writerConfig.FilePath)
 	if err != nil {
 		log.Fatalf("FileWriter ERROR while trying to open file: %v", err)

output/stdout.go

@@ -5,13 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"sync"
 )
 
 type StdoutWriter struct{}
 
-func (s *StdoutWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup) {
-	defer wg.Done()
+func (s *StdoutWriter) Write(items chan map[string]interface{}) {
 	for item := range items {
 		// We cannot use the following line of code because it automatically replaces certain html characters
 		// with the corresponding Unicode replacement rune.

output/writer.go

@@ -1,11 +1,9 @@
 package output
 
-import "sync"
-
 type Writer interface {
 	// if a writer encounters a fatal error it should call log.Fatalf
 	// to prevent the crawler from uselessly continuing to run.
-	Write(itemsList chan map[string]interface{}, wg *sync.WaitGroup)
+	Write(itemsList chan map[string]interface{})
 }
 
 // .WriterConfig defines the necessary paramters to make a new writer