adding concurrency for performance

pkg/traversal/bfs.go

@@ -3,6 +3,7 @@ package traversal
 import (
 	"fmt"
 	"log"
+	"sync"
 
 	"github.com/karthikkalarikal/Qube/models"
 	"github.com/karthikkalarikal/Qube/pkg/util"
@@ -21,77 +22,98 @@ func NewNode(start, target string) {
 }
 
 func bfs(start, end string) *Node {
-	visited := make(map[string]bool)
+	var visited sync.Map
+	mu := sync.Mutex{}
+	wg := sync.WaitGroup{}
+	gofer := make(chan struct{}, 10)
 	queue := make([]Node, 1)
 	queue[0] = Node{Entity: start, Type: "person"}
 
 	for len(queue) > 0 {
-		current := queue[0]
-		queue = queue[1:]
-		log.Println(current.Type)
-		if current.Entity == end {
-			return &current
-		}
-		visited[current.Entity] = true
-
-		if current.Type == "person" {
-			var person models.Actor
-
-			if err := util.GetByURL(current.Entity, &person); err != nil {
-				visited[current.Entity] = true
-				log.Printf("Skipping inaccessible node: %s (%v)", current.Entity, err)
-				continue
+		l := len(queue)
+		for i := 0; i < l; i++ {
+
+			current := queue[0]
+			queue = queue[1:]
+			log.Println(current.Entity, " ", current.Type)
+			if current.Entity == end {
+				return &current
 			}
-
-			for _, credit := range person.Movies {
-				node := Node{
-					Entity: credit.URL,
-					Type:   "movie",
-					From:   &current,
-					Link:   "Movie: " + credit.Name + " (" + credit.Role + ")",
-				}
-
-				if !visited[credit.URL] {
-					queue = append(queue, node)
+			visited.Store(current.Entity, true)
+			wg.Add(1)
+			gofer <- struct{}{}
+			go func(current Node) {
+				defer func() {
+					wg.Done()
+					<-gofer
+				}()
+				if current.Type == "person" {
+					var person models.Actor
+
+					if err := util.GetByURL(current.Entity, &person); err != nil {
+						visited.Store(current.Entity, true)
+						log.Printf("Skipping inaccessible node: %s (%v)", current.Entity, err)
+
+					}
+
+					for _, credit := range person.Movies {
+						node := Node{
+							Entity: credit.URL,
+							Type:   "movie",
+							From:   &current,
+							Link:   "Movie: " + credit.Name + " (" + credit.Role + ")",
+						}
+
+						if _, ok := visited.Load(credit.URL); !ok {
+							mu.Lock()
+							queue = append(queue, node)
+							mu.Unlock()
+						}
+					}
+				} else {
+					var movie models.Movie
+
+					if err := util.GetByURL(current.Entity, &movie); err != nil {
+						visited.Store(current.Entity, true)
+						log.Printf("Skipping inaccessible node: %s (%v)", current.Entity, err)
+
+					}
+
+					log.Println(movie.URL)
+					for _, cast := range movie.Cast {
+
+						node := Node{
+							Entity: cast.URL,
+							Type:   "person",
+							From:   &current,
+							Link:   "Cast: " + cast.Name + " (" + cast.Role + ")",
+						}
+						if _, ok := visited.Load(cast.URL); !ok {
+							mu.Lock()
+							queue = append(queue, node)
+							mu.Unlock()
+						}
+
+					}
+					for _, crew := range movie.Crew {
+
+						node := Node{
+							Entity: crew.URL,
+							Type:   "person",
+							From:   &current,
+							Link:   "Crew: " + crew.Name + " (" + crew.Role + ")",
+						}
+
+						if _, ok := visited.Load(crew.URL); !ok {
+							mu.Lock()
+							queue = append(queue, node)
+							mu.Unlock()
+						}
+					}
 				}
-			}
-		} else {
-			var movie models.Movie
-
-			if err := util.GetByURL(current.Entity, &movie); err != nil {
-				visited[current.Entity] = true
-				log.Printf("Skipping inaccessible node: %s (%v)", current.Entity, err)
-				continue
-			}
-
-			log.Println(movie.URL)
-			for _, cast := range movie.Cast {
-
-				node := Node{
-					Entity: cast.URL,
-					Type:   "person",
-					From:   &current,
-					Link:   "Cast: " + cast.Name + " (" + cast.Role + ")",
-				}
-				if !visited[cast.URL] {
-					queue = append(queue, node)
-				}
-
-			}
-			for _, crew := range movie.Crew {
-
-				node := Node{
-					Entity: crew.URL,
-					Type:   "person",
-					From:   &current,
-					Link:   "Crew: " + crew.Name + " (" + crew.Role + ")",
-				}
-
-				if !visited[crew.URL] {
-					queue = append(queue, node)
-				}
-			}
+			}(current)
 		}
+		wg.Wait()
 
 	}
 	return nil

pkg/util/util.go

@@ -24,7 +24,7 @@ func GetByURL(url string, target any) error {
 		return err
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusForbidden {
+	if resp.StatusCode == http.StatusForbidden {
 
 		return fmt.Errorf("access denied for URL: %s", url)