implement caching layer for digger config (#1825)

* implement caching layer for digger config

.github/workflows/backend_test.yml

@@ -35,7 +35,9 @@ jobs:
         working-directory: backend
 
       - name: Test
-        run: go test -v ./...
+        run: |
+          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+          go test -json ./... | gotestfmt
         env:
           GITHUB_PAT_TOKEN: ${{ secrets.TOKEN_GITHUB }}
         working-directory: backend

backend/bootstrap/main.go

@@ -221,6 +221,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 	runsApiGroup.GET("/:run_id", controllers.RunDetails)
 	runsApiGroup.POST("/:run_id/approve", controllers.ApproveRun)
 
+	// internal endpoints not meant to be exposed to public and protected behing webhook secret
+	r.POST("_internal/update_repo_cache", middleware.WebhookAuth(), diggerController.UpdateRepoCache)
+
 	fronteggWebhookProcessor.POST("/create-org-from-frontegg", controllers.CreateFronteggOrgFromWebhook)
 
 	return r

backend/controllers/cache.go

@@ -0,0 +1,101 @@
+package controllers
+
+import (
+	"fmt"
+	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
+	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
+	"github.com/gin-gonic/gin"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+)
+
+func (d DiggerController) UpdateRepoCache(c *gin.Context) {
+	type UpdateCacheRequest struct {
+		RepoFullName   string `json:"repo_full_name"`
+		Branch         string `json:"branch"`
+		OrgId          uint   `json:"org_id"`
+		InstallationId int64  `json:"installation_id"`
+	}
+
+	var request UpdateCacheRequest
+	err := c.BindJSON(&request)
+	if err != nil {
+		log.Printf("Error binding JSON: %v", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error binding JSON"})
+		return
+	}
+
+	repoFullName := request.RepoFullName
+	installationId := request.InstallationId
+	link, err := models.DB.GetGithubAppInstallationLink(installationId)
+	if err != nil {
+		log.Printf("could not installation link: %v", err)
+		c.String(500, fmt.Sprintf("coulnt not find installation link %v %v", repoFullName, installationId))
+		return
+
+	}
+	orgId := link.OrganisationId
+
+	log.Printf("the org id is %v", orgId)
+
+	repoOwner, repoName, _ := strings.Cut(repoFullName, "/")
+	repoDiggerName := strings.ReplaceAll(repoFullName, "/", "-")
+
+	repo, err := models.DB.GetRepo(orgId, repoDiggerName)
+	if err != nil {
+		log.Printf("could not get repo: %v", err)
+		c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
+		return
+	}
+
+	cloneUrl := repo.RepoUrl
+	branch := request.Branch
+
+	//ghInstallation, err := models.DB.GetInstallationForRepo(repoFullName)
+	//if err != nil {
+	//	log.Printf("could not get repo: %v", err)
+	//	c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
+	//	return
+	//}
+
+	_, token, err := utils.GetGithubService(d.GithubClientProvider, installationId, repoFullName, repoOwner, repoName)
+	if err != nil {
+		log.Printf("could not get github service :%v", err)
+		c.String(500, fmt.Sprintf("could not get github service %v %v", repoFullName, orgId))
+		return
+	}
+
+	var diggerYmlStr string
+	var config *dg_configuration.DiggerConfig
+
+	// update the cache here, do it async for immediate response
+	go func() {
+		err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+			diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
+			diggerYmlStr = string(diggerYmlBytes)
+			config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
+			if err != nil {
+				log.Printf("Error loading digger config: %v", err)
+				return err
+			}
+			return nil
+		})
+
+		if err != nil {
+			log.Printf("could not load digger config :%v", err)
+			return
+		}
+		_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config)
+		if err != nil {
+			log.Printf("could upadate repo cache :%v", err)
+			return
+
+		}
+	}()
+
+	c.String(200, "successfully submitted cache for processing, check backend logs for progress")
+}

backend/controllers/github.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -53,7 +54,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 	log.Printf("GithubAppWebHook")
 
 	appID := c.GetHeader("X-GitHub-Hook-Installation-Target-ID")
-	log.Printf("app id from header is: %v", appID)
+
 	_, _, webhookSecret, _, err := d.GithubClientProvider.FetchCredentials(appID)
 
 	payload, err := github.ValidatePayload(c.Request, []byte(webhookSecret))
@@ -316,6 +317,10 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	commitSha := payload.PullRequest.Head.GetSHA()
 	branch := payload.PullRequest.Head.GetRef()
 	action := *payload.Action
+	labels := payload.PullRequest.Labels
+	prLabelsStr := lo.Map(labels, func(label *github.Label, i int) string {
+		return *label.Name
+	})
 
 	link, err := models.DB.GetGithubAppInstallationLink(installationId)
 	if err != nil {
@@ -361,7 +366,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		}
 	}
 
-	diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
+	diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, organisationId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
 	if err != nil {
 		log.Printf("getDiggerConfigForPR error: %v", err)
 		return fmt.Errorf("error getting digger config")
@@ -563,7 +568,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 }
 
 // TODO: Refactor this func to receive ghService as input
-func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
+func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []string, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
 	ghService, _, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
 	if err != nil {
 		log.Printf("Error getting github service: %v", err)
@@ -583,6 +588,17 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
 		return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting changed files")
 	}
 
+	// check if items should be loaded from cache
+	if val, _ := os.LookupEnv("DIGGER_CONFIG_REPO_CACHE_ENABLED"); val == "1" && !slices.Contains(prLabels, "digger:no-cache") {
+		diggerYmlStr, config, dependencyGraph, err := retrieveConfigFromCache(orgId, repoFullName)
+		if err != nil {
+			log.Printf("could not load from cache")
+		} else {
+			log.Printf("successfully loaded from cache")
+			return diggerYmlStr, ghService, config, *dependencyGraph, &prBranch, &prCommitSha, nil
+		}
+	}
+
 	diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles)
 	if err != nil {
 		log.Printf("Error loading digger.yml: %v", err)
@@ -593,6 +609,28 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
 	return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, nil
 }
 
+func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], error) {
+	repoCache, err := models.DB.GetRepoCache(orgId, repoFullName)
+	if err != nil {
+		log.Printf("Error: failed to load repoCache, going to try live load %v", err)
+		return "", nil, nil, fmt.Errorf("")
+	}
+	var config dg_configuration.DiggerConfig
+	err = json.Unmarshal(repoCache.DiggerConfig, &config)
+	if err != nil {
+		log.Printf("Error: failed to load repoCache unmarshall config %v", err)
+		return "", nil, nil, fmt.Errorf("failed to load repoCache unmarshall config %v", err)
+	}
+
+	projectsGraph, err := dg_configuration.CreateProjectDependencyGraph(config.Projects)
+	if err != nil {
+		log.Printf("error retrieving graph of dependencies: %v", err)
+		return "", nil, nil, fmt.Errorf("error retrieving graph of dependencies: %v", err)
+	}
+
+	return repoCache.DiggerYmlStr, &config, &projectsGraph, nil
+}
+
 func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {
 	link, err := models.DB.GetGithubAppInstallationLink(installationId)
 	if err != nil {
@@ -634,6 +672,10 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	commentBody := *payload.Comment.Body
 	defaultBranch := *payload.Repo.DefaultBranch
 	isPullRequest := payload.Issue.IsPullRequest()
+	labels := payload.Issue.Labels
+	prLabelsStr := lo.Map(labels, func(label *github.Label, i int) string {
+		return *label.Name
+	})
 
 	if !isPullRequest {
 		log.Printf("comment not on pullrequest, ignroning")
@@ -671,7 +713,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		}
 	}
 
-	diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
+	diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
 	if err != nil {
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not load digger config, error: %v", err))
 		log.Printf("getDiggerConfigForPR error: %v", err)