Adjust the logging for the new recursive include parsing

The version v0.13.2 has a lot of errors complaining that the code is not
requesting valid file paths and the logging was not disclosing enough
information to track down the actual error.

Now the GitLab client gets handed down a logger to be able to have some
trace logging on the requests, this commit only adds a trace log to the
GetRawFileFromProject requestURL to further debug potential errors.

The error logging during the handling of an input now logs the actual
path of the project it failed during.

internal/crawler/crawler.go

@@ -37,7 +37,7 @@ func New(cfg *Config, logger zerolog.Logger, store storage.Storage) (*Crawler, e
 		RateLimiter: rate.NewLimiter(rate.Limit(cfg.GitlabMaxRPS), cfg.GitlabMaxRPS),
 	}
 
-	gitlabClient := gitlab.NewClient(cfg.GitlabHost, cfg.GitlabToken, httpClient)
+	gitlabClient := gitlab.NewClient(cfg.GitlabHost, cfg.GitlabToken, httpClient, logger)
 
 	return &Crawler{
 		config:       cfg,
@@ -124,7 +124,10 @@ func (c *Crawler) updateProjectInGraph(ctx context.Context, project gitlab.Proje
 
 		err := c.handleIncludes(ctx, project, gitlabCIFileName)
 		if err != nil {
-			c.logger.Err(err).Msg("failed to handle all includes")
+			c.logger.Error().
+				Err(err).
+				Str("Project", project.PathWithNamespace).
+				Msg("failed to handle all includes")
 		}
 		return nil
 	}

internal/gitlab/gitlab.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/rs/zerolog"
 	"io"
 	"net/http"
 	"net/url"
@@ -25,6 +26,7 @@ type Client struct {
 	Host     string
 	Token    string
 	HTTPDoer HTTPDoer
+	Logger   zerolog.Logger
 }
 
 type HTTPDoer interface {
@@ -42,11 +44,12 @@ type Project struct {
 // NewClient sets up a client struct for all relevant GitLab auth
 // you can give it a custom http.Client as well for things like
 // timeouts.
-func NewClient(host, token string, httpDoer HTTPDoer) *Client {
+func NewClient(host, token string, httpDoer HTTPDoer, logger zerolog.Logger) *Client {
 	return &Client{
 		Host:     host,
 		Token:    token,
 		HTTPDoer: httpDoer,
+		Logger:   logger,
 	}
 }
 
@@ -170,6 +173,7 @@ func (c *Client) GetRawFileFromProject(ctx context.Context, projectID int, fileN
 	queryParams := url.Values{}
 	queryParams.Add("ref", ref)
 	requestURL := fmt.Sprintf("%s/%s/projects/%d/repository/files/%s/raw?%s", c.Host, gitLabAPIPath, projectID, url.PathEscape(fileName), queryParams.Encode())
+	c.Logger.Trace().Str("RequestURL", requestURL).Msg("requesting raw file from GitLab")
 
 	resp, err := c.callGitLabAPI(ctx, requestURL)
 	if err != nil {