From 708608d5018d09b46f2a7cec6a413c49d27e2541 Mon Sep 17 00:00:00 2001 From: Andrei Predoiu <5424946+Andrei-Predoiu@users.noreply.github.com> Date: Mon, 22 Apr 2024 10:58:33 +0200 Subject: [PATCH] ES-2153 - Added support for multiple files and for updating default parameters (#426) * Changed how directory works, now it points to the circleci directory - BREAKING change Added defaults for directory and schedule * max 100 files * Update README.md --------- Co-authored-by: Lasse Gaardsholt --- .gitignore | 2 + README.md | 47 ++++--- api/config_check.go | 2 +- api/controller.go | 16 ++- circleci/docker.go | 57 +++++--- circleci/orb.go | 45 ++++-- circleci/update.go | 134 +++++++++++++++--- circleci/update_test.go | 20 +-- config/config.go | 12 +- dependabot/dependabot.go | 294 ++++++++++++++++++++++++--------------- gh/commit.go | 15 +- gh/repos.go | 5 +- go.mod | 2 + go.sum | 8 +- main.go | 4 +- 15 files changed, 441 insertions(+), 222 deletions(-) diff --git a/.gitignore b/.gitignore index a7eadbe5..71f81db0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ db-secrets real-secrets* secrets* app-secrets + +/.idea diff --git a/README.md b/README.md index 4b745c3f..87778e55 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ We have created this as at the time of creation it was nearly impossible to get
## Getting Started + 1. Install the `dependabot-circleci` [GitHub App](https://github.com/apps/dependabot-circleci) in your organization. 2. You enable `dependabot-circleci` on specific repositories by creating a `dependabot-circleci.yml` configuration file in your repository's `.github` directory. `dependabot-circleci` then raise pull requests to keep the dependencies you configure up-to-date. @@ -37,42 +38,46 @@ reviewers: - github_username # for a single user - org/team_name # for a whole team (nested teams is the same syntax org/team_name) target-branch: main -directory: "/template" # Used if .github directory is nested inside another directory +directory: "/.circleci/config.yml" # Folder where the circleci config files are located schedule: "monthly" # Options are (daily, weekly, monthly) ``` +dependabot-circleci will recursively scan all the files and folders in the directory specified in the `directory` field for CircleCI config files. If it finds any outdated dependencies, it will raise pull requests against the target branch specified in the `target-branch` field. dependabot-circleci will scan a maximum of 100 entities(folders or yaml/yml files). + ---
## Configuration options for dependency updates -The `dependabot-circleci` configuration file, dependabot-circleci.yml, uses YAML syntax. -You must store this file in the .github directory of your repository. -| Option | Required | Description | Default | -| :-------------------------------- | :------: | :------------------------------------- | -------------------------- | -| [`assignees`](#assignees) | | Assignees to set on pull requests | n/a | -| [`labels`](#labels) | | Labels to set on pull requests | n/a | -| [`reviewers`](#reviewers) | | Reviewers to set on pull requests | n/a | -| [`target-branch`](#target-branch) | | Branch to create pull requests against | Default branch in the repo | -| [`directory`](#directory) | | Location of .github directory | Root of repo | -| [`schedule`](#schedule) | | When to look for updates | daily | +The `dependabot-circleci` configuration file, dependabot-circleci.yml, uses YAML syntax. +You must store this file in the .github directory of your repository. +| Option | Required | Description | Default | +|:----------------------------------|:--------:|:-----------------------------------------------------------------------------------------------|----------------------------| +| [`assignees`](#assignees) | | Assignees to set on pull requests | n/a | +| [`labels`](#labels) | | Labels to set on pull requests | n/a | +| [`reviewers`](#reviewers) | | Reviewers to set on pull requests | n/a | +| [`target-branch`](#target-branch) | | Branch to create pull requests against | Default branch in the repo | +| [`directory`](#directory) | | Path to the circleci config file, or folder to be scanned | `/.circleci/config.yml` | +| [`schedule`](#schedule) | | When to look for updates | daily | ---
## Contributing + We are open for issues, pull requests etc. ## Running locally + 1. Clone the repository 2. Make sure to have your secrets file in place - 2.1 BESTSELLER folks can use Harpocrates to get them from Vault. + 2.1 BESTSELLER folks can use Harpocrates to get them from Vault. ```bash harpocrates -f secrets-local.yaml --vault-token $(vault token create -format=json | jq -r '.auth.client_token') ``` - 2.2 Others will have to fill out this template in any other way. + 2.2 Others will have to fill out this template in any other way. ```json { "datadog": { @@ -80,7 +85,7 @@ We are open for issues, pull requests etc. }, "github": { "app": { - "integration_id": , + "integration_id": "", "private_key": "", "webhook_secret": "" }, @@ -103,7 +108,17 @@ We are open for issues, pull requests etc. } ``` 3. Run `dependabot-circleci` by using Docker compose - > `--build` will ensure that the latest version of the code is used + > `--build` will ensure that the latest version of the code is used ```bash docker-compose up --build - ``` \ No newline at end of file + ``` +4. Test worker by sending a POST request to `http://localhost:3000/worker` with the following payload + ```bash + curl --request POST \ + --url http://localhost:3000/start \ + --header 'Content-Type: application/json' \ + --data '{"Org":"BESTSELLER","Repos": ["dependabot-circleci"]}' + ``` +5. If you want to debug the worker without docker: + 1. Add the env vars from the docker-compose file to your local environment to match the worker + 2. Run/Debug in your IDE with the `-worker` flag \ No newline at end of file diff --git a/api/config_check.go b/api/config_check.go index f929d0df..fc409178 100644 --- a/api/config_check.go +++ b/api/config_check.go @@ -59,7 +59,7 @@ func (h *ConfigCheckHandler) Handle(ctx context.Context, eventType, deliveryID s } // get content - content, _, err := gh.GetRepoContent(ctx, client, Githubinfo.Owner, Githubinfo.RepoName, ".github/dependabot-circleci.yml", commitSHA) + content, _, err := gh.GetRepoFileBytes(ctx, client, Githubinfo.Owner, Githubinfo.RepoName, ".github/dependabot-circleci.yml", commitSHA) if err != nil { log.Debug().Err(err).Msg("could not read content of repository") return nil // we dont care diff --git a/api/controller.go b/api/controller.go index cc4bb461..13f93248 100644 --- a/api/controller.go +++ b/api/controller.go @@ -28,7 +28,7 @@ type WorkerPayload struct { var wg sync.WaitGroup -func controllerHandler(w http.ResponseWriter, r *http.Request) { +func controllerHandler(w http.ResponseWriter, _ *http.Request) { log.Debug().Msg("controllerHandler called") orgs, err := pullRepos() @@ -80,14 +80,16 @@ func shouldRun(schedule string) bool { // check if an update should be run t := time.Now() schedule = strings.ToLower(schedule) - if schedule == "monthly" { - return (t.Day() == 1) - } else if schedule == "weekly" { - return (t.Weekday() == 1) - } else if schedule == "daily" || schedule == "" { + switch schedule { + case "daily", "": return true + case "weekly": + return t.Weekday() == 1 + case "monthly": + return t.Day() == 1 + default: + return false } - return false } // PostJSON posts the structs as json to the specified url diff --git a/circleci/docker.go b/circleci/docker.go index 71c6f2d9..8c70676d 100644 --- a/circleci/docker.go +++ b/circleci/docker.go @@ -13,7 +13,7 @@ import ( "gopkg.in/yaml.v3" ) -func extractImages(images []*yaml.Node) map[string]*yaml.Node { +func extractImages(images []*yaml.Node, parameters *map[string]*yaml.Node) map[string]*yaml.Node { updates := map[string]*yaml.Node{} for i := 0; i < len(images); i++ { image := images[i] @@ -21,16 +21,16 @@ func extractImages(images []*yaml.Node) map[string]*yaml.Node { image = images[i+1] log.Debug().Msg(fmt.Sprintf("current image version: %s", image.Value)) - imageVersion := findNewestDockerVersion(image.Value) - log.Debug().Msg(fmt.Sprintf("new image version: %s", imageVersion)) + imageName, currentTag, newestTag := findNewestDockerVersion(image.Value, parameters) + log.Debug().Msg(fmt.Sprintf("new image version: %s:%s", imageName, newestTag)) - if image.Value != imageVersion { + if currentTag != newestTag { oldVersion := image.Value - image.Value = imageVersion + image.Value = newestTag updates[oldVersion] = image } } - baah := extractImages(image.Content) + baah := extractImages(image.Content, parameters) for k, v := range baah { updates[k] = v } @@ -38,31 +38,46 @@ func extractImages(images []*yaml.Node) map[string]*yaml.Node { return updates } -func findNewestDockerVersion(currentVersion string) string { +func findNewestDockerVersion(currentVersion string, parameters *map[string]*yaml.Node) (imageName, currentTag, newestTag string) { current := strings.Split(currentVersion, ":") // check if image has no version tag if len(current) == 1 { - return currentVersion + return currentVersion, "", "" } // check if tag is latest if strings.ToLower(current[1]) == "latest" { - return currentVersion + return current[0], current[1], current[1] + } + imageName = current[0] + currentTag = current[1] + + if newVersion, hit := cache[currentVersion]; hit { + log.Debug().Msgf("Using cached version for image: %s", currentVersion) + return imageName, currentTag, newVersion + } + + if param := ExtractParameterName(currentVersion); len(param) > 0 { + paramDefault, found := (*parameters)[param] + if !found { + log.Debug().Msgf("Parameter %s not found in parameters", param) + return imageName, currentTag, currentTag + } + currentTag = paramDefault.Value } // fix this shit - tags, err := getTags(currentVersion) + tags, err := getTags(imageName) if err != nil { log.Debug().Err(err) - return currentVersion + return imageName, currentTag, currentTag } - versionParts := splitVersion(current[1]) + versionParts := splitVersion(currentTag) if len(versionParts) == 0 { - return currentVersion + return imageName, currentTag, currentTag } - var newTagsList []string for _, tag := range tags { aa := splitVersion(tag) @@ -95,10 +110,12 @@ func findNewestDockerVersion(currentVersion string) string { currentv, _ := version.NewVersion(versionParts["version"]) if currentv.GreaterThan(newest) { - return currentVersion + cache[currentVersion] = currentTag + return imageName, currentTag, currentTag } - - return fmt.Sprintf("%s:%s", current[0], newest.Original()) + newVersion := newest.Original() + cache[currentVersion] = newVersion + return imageName, currentTag, newVersion } func getTags(circleciTag string) ([]string, error) { @@ -152,9 +169,9 @@ func splitVersion(version string) map[string]string { } matches := myExp.SubexpNames() - for i, name := range matches { - if i != 0 && name != "" && match[i] != "" { - result[name] = match[i] + for i, imgName := range matches { + if i != 0 && imgName != "" && match[i] != "" { + result[imgName] = match[i] } } diff --git a/circleci/orb.go b/circleci/orb.go index 9339113e..0e62858e 100644 --- a/circleci/orb.go +++ b/circleci/orb.go @@ -12,31 +12,44 @@ import ( "gopkg.in/yaml.v3" ) -func extractOrbs(orbs []*yaml.Node) map[string]*yaml.Node { +func extractOrbs(orbs []*yaml.Node, parameters *map[string]*yaml.Node) map[string]*yaml.Node { updates := map[string]*yaml.Node{} for i := 0; i < len(orbs); i = i + 2 { orb := orbs[i+1] log.Debug().Msg(fmt.Sprintf("current orb version: %s", orb.Value)) - orbVersion := findNewestOrbVersion(orb.Value) - log.Debug().Msg(fmt.Sprintf("new orb version: %s", orbVersion)) + orbRoot, currentVer, newestVer := findNewestOrbVersion(orb.Value, parameters) + log.Debug().Msg(fmt.Sprintf("new orb version: %s@%s", orbRoot, newestVer)) - if orb.Value != orbVersion { + if currentVer != newestVer { oldVersion := orb.Value - orb.Value = orbVersion + orb.Value = newestVer updates[oldVersion] = orb } } return updates } -func findNewestOrbVersion(orb string) string { - - orbSplitString := strings.Split(orb, "@") - +func findNewestOrbVersion(currentVersion string, parameters *map[string]*yaml.Node) (orbName, currentTag, newestTag string) { + orbSplitString := strings.Split(currentVersion, "@") // check if orb is always updated if orbSplitString[1] == "volatile" || strings.HasPrefix(orbSplitString[1], "dev:") { - return orbSplitString[1] + return orbSplitString[0], orbSplitString[1], orbSplitString[1] + } + orbName = orbSplitString[0] + currentTag = orbSplitString[1] + + if newestTag, hit := cache[currentVersion]; hit { + log.Debug().Msgf("Using cached version for orb: %s - %s", orbName, newestTag) + return orbName, currentTag, newestTag + } + if param := ExtractParameterName(currentVersion); len(param) > 0 { + paramDefault, found := (*parameters)[param] + if !found { + log.Debug().Msgf("Parameter %s not found in parameters", param) + return orbName, currentTag, currentTag + } + currentTag = paramDefault.Value } CCIApiToken := "" @@ -48,11 +61,17 @@ func findNewestOrbVersion(orb string) string { client := graphql.NewClient(http.DefaultClient, "https://circleci.com/", "graphql-unstable", CCIApiToken, false) // if requests fails, return current version - orbInfo, err := api.OrbInfo(client, orbSplitString[0]) + orbInfo, err := api.OrbInfo(client, orbName) if err != nil { log.Error().Err(err).Msgf("error finding latests orb version failed for orb: %s", orbSplitString[0]) - return fmt.Sprintf("%s@%s", orbSplitString[0], orbSplitString[1]) + return orbName, currentTag, currentTag + } + + if len(orbInfo.Orb.HighestVersion) == 0 { + cache[currentVersion] = currentTag + return orbName, currentTag, currentTag } - return fmt.Sprintf("%s@%s", orbSplitString[0], orbInfo.Orb.HighestVersion) + cache[currentVersion] = orbInfo.Orb.HighestVersion + return orbName, currentTag, orbInfo.Orb.HighestVersion } diff --git a/circleci/update.go b/circleci/update.go index f2d2edf8..2e6dda47 100644 --- a/circleci/update.go +++ b/circleci/update.go @@ -1,31 +1,58 @@ package circleci import ( + "regexp" "strings" "gopkg.in/yaml.v3" ) -func getDockerUpdates(node *yaml.Node) map[string]*yaml.Node { +type Update struct { + Type string + CurrentName string + FileUpdates map[string]FileUpdate +} + +func (update Update) SplitCurrentVersion() []string { + var oldVersion []string + var separator string + if update.Type == "orb" { + separator = "@" + } else { + separator = ":" + } + oldVersion = strings.Split(update.CurrentName, separator) + return oldVersion +} + +type FileUpdate struct { + SHA *string + Content *string + Parameters *map[string]*yaml.Node + Node *yaml.Node +} + +var cache = map[string]string{} + +func getDockerUpdates(node *yaml.Node, parameters *map[string]*yaml.Node) map[string]*yaml.Node { updates := map[string]*yaml.Node{} for i, nextHole := range node.Content { - if nextHole.Value == "executors" || nextHole.Value == "jobs" { - + switch nextHole.Value { + case "executors", "jobs", "docker": // check if there is a docker image if i+1 >= len(node.Content) { return updates } - dockers := node.Content[i+1] - updates := extractImages(dockers.Content) + updates := extractImages(dockers.Content, parameters) for k, v := range updates { updates[k] = v } return updates } - next := getDockerUpdates(nextHole) + next := getDockerUpdates(nextHole, parameters) for k, v := range next { updates[k] = v } @@ -33,20 +60,47 @@ func getDockerUpdates(node *yaml.Node) map[string]*yaml.Node { return updates } -func getOrbUpdates(node *yaml.Node) map[string]*yaml.Node { + +// Recurse until we find a block called parameters then we extract a map with the default values +func extractParameters(yamlNode []*yaml.Node) map[string]*yaml.Node { + for a, nextHole := range yamlNode { + if nextHole.Value == "parameters" { + if parametersBlock := yamlNode[a+1].Content; len(parametersBlock) > 0 { + results := map[string]*yaml.Node{} + for pi, param := range parametersBlock { + if len(param.Value) > 0 { + for c, k := range parametersBlock[pi+1].Content { + if k.Value == "default" { + results[param.Value] = parametersBlock[pi+1].Content[c+1] + } + } + } + } + return results + } + } else if len(nextHole.Content) > 0 { + if results := extractParameters(nextHole.Content); results != nil { + return results + } + } + } + return nil +} + +func getOrbUpdates(node *yaml.Node, parameters *map[string]*yaml.Node) map[string]*yaml.Node { updates := map[string]*yaml.Node{} for i, nextHole := range node.Content { if nextHole.Value == "orbs" { orbs := node.Content[i+1] - updates := extractOrbs(orbs.Content) + updates := extractOrbs(orbs.Content, parameters) for k, v := range updates { updates[k] = v } return updates } - next := getOrbUpdates(nextHole) + next := getOrbUpdates(nextHole, parameters) for k, v := range next { updates[k] = v } @@ -55,20 +109,64 @@ func getOrbUpdates(node *yaml.Node) map[string]*yaml.Node { return updates } -// GetUpdates returns a list of updated yaml nodes -func GetUpdates(node *yaml.Node) (map[string]*yaml.Node, map[string]*yaml.Node) { - return getOrbUpdates(node), getDockerUpdates(node) +// ScanFileUpdates returns a map of updates found in a file, the key is the original version +func ScanFileUpdates(updates *map[string]Update, content, path, SHA *string) error { + // unmarshal + var nodeContent yaml.Node + err := yaml.Unmarshal([]byte(*content), &nodeContent) + if err != nil { + return err + } + + parameters := extractParameters(nodeContent.Content) + for k, orbUpdate := range getOrbUpdates(&nodeContent, ¶meters) { + if _, contains := (*updates)[orbUpdate.Value]; !contains { + (*updates)[orbUpdate.Value] = Update{ + Type: "orb", + CurrentName: k, + FileUpdates: make(map[string]FileUpdate), + } + } + (*updates)[orbUpdate.Value].FileUpdates[*path] = FileUpdate{ + SHA: SHA, + Node: orbUpdate, + Content: content, + Parameters: ¶meters, + } + } + + for k, dockerUpdate := range getDockerUpdates(&nodeContent, ¶meters) { + if _, contains := (*updates)[dockerUpdate.Value]; !contains { + (*updates)[dockerUpdate.Value] = Update{ + Type: "docker", + CurrentName: k, + FileUpdates: make(map[string]FileUpdate), + } + } + (*updates)[dockerUpdate.Value].FileUpdates[*path] = FileUpdate{ + SHA: SHA, + Node: dockerUpdate, + Content: content, + Parameters: ¶meters, + } + } + return nil } // ReplaceVersion replaces a specific line in the yaml -func ReplaceVersion(orb *yaml.Node, oldVersion string, content string) string { - +func ReplaceVersion(lineNumber int, oldVersion, newVersion, content string) string { lines := strings.Split(content, "\n") - lineNumber := orb.Line + -1 theLine := lines[lineNumber] - lines[lineNumber] = strings.ReplaceAll(theLine, oldVersion, orb.Value) - + lines[lineNumber] = strings.ReplaceAll(theLine, oldVersion, newVersion) output := strings.Join(lines, "\n") - return output } + +func ExtractParameterName(param string) string { + r := regexp.MustCompile(`<<\s*parameters\.(\w+)\s*>>`) + match := r.FindStringSubmatch(param) + if len(match) > 1 { + return match[1] + } + return "" +} diff --git a/circleci/update_test.go b/circleci/update_test.go index 6951e7f4..b13598a0 100644 --- a/circleci/update_test.go +++ b/circleci/update_test.go @@ -8,16 +8,15 @@ import ( "testing" "github.com/rs/zerolog/log" - "gopkg.in/yaml.v3" ) -func getTestCases() map[string]*yaml.Node { +func getTestCases() map[string]*string { path := "../.test_cases" files, err := os.ReadDir(path) if err != nil { log.Fatal().Err(err) } - result := make(map[string]*yaml.Node, len(files)) + result := make(map[string]*string, len(files)) for _, f := range files { fileName := f.Name() ext := strings.ToLower(filepath.Ext(fileName)) @@ -28,25 +27,20 @@ func getTestCases() map[string]*yaml.Node { fmt.Println(f.Name()) content, _ := os.ReadFile(filePath) - var cciconfig yaml.Node - err = yaml.Unmarshal(content, &cciconfig) - if err != nil { - continue - } - result[fileName] = &cciconfig + contentString := string(content) + result[fileName] = &contentString } return result } func TestGetUpdates(t *testing.T) { tests := getTestCases() + SHA := "ABC" + updates := map[string]Update{} for k, v := range tests { t.Run(k, func(t *testing.T) { - GetUpdates(v) - // if got := GetUpdates(v); !reflect.DeepEqual(got, tt.want) { - // t.Errorf("GetUpdates() = %v, want %v", got, tt.want) - // } + ScanFileUpdates(&updates, v, &k, &SHA) }) } } diff --git a/config/config.go b/config/config.go index 098838d8..a540ddd0 100644 --- a/config/config.go +++ b/config/config.go @@ -33,7 +33,7 @@ type BestsellerSpecificConfig struct { Running bool } -// DBConfig contains global db config +// DBConfigSpec contains global db config type DBConfigSpec struct { ConnectionName string `yaml:"connection_name"` ConnectionString string `yaml:"connection_string"` @@ -43,7 +43,7 @@ type DBConfigSpec struct { Username string `yaml:"username"` } -// RepoConfig contains specific config for each repos +// RepoConfig contains specific config for each repo type RepoConfig struct { TargetBranch string `yaml:"target-branch,omitempty"` Reviewers []string `yaml:"reviewers,omitempty"` @@ -83,8 +83,8 @@ func ReadDBConfig(secrets []byte) error { // ReadRepoConfig reads a yaml file func ReadRepoConfig(content []byte) (*RepoConfig, error) { - - var repoConfig RepoConfig + // default values setup here + repoConfig := RepoConfig{Directory: ".circleci/config.yml", Schedule: "daily"} if err := yaml.UnmarshalStrict(content, &repoConfig); err != nil { return nil, errors.Wrap(err, "failed parsing repository configuration file") @@ -94,11 +94,11 @@ func ReadRepoConfig(content []byte) (*RepoConfig, error) { } // IsValid checks if Repoconfig is valid -func (r RepoConfig) IsValid() error { +func (rc RepoConfig) IsValid() error { var errMsg []string // check schedule - switch strings.ToLower(r.Schedule) { + switch strings.ToLower(rc.Schedule) { case "daily", "weekly", "monthly", "": default: diff --git a/dependabot/dependabot.go b/dependabot/dependabot.go index 95bb69fe..fc57da68 100644 --- a/dependabot/dependabot.go +++ b/dependabot/dependabot.go @@ -2,8 +2,11 @@ package dependabot import ( "context" + "errors" "fmt" "strings" + "sync" + "sync/atomic" "github.com/BESTSELLER/dependabot-circleci/circleci" "github.com/BESTSELLER/dependabot-circleci/config" @@ -11,14 +14,21 @@ import ( "github.com/BESTSELLER/dependabot-circleci/gh" "github.com/google/go-github/v60/github" "github.com/rs/zerolog/log" - "gopkg.in/yaml.v3" ) +type RepoInfo struct { + repoConfig *config.RepoConfig + repoOwner string + repoDefaultBranch string + targetBranch string + repoName string +} + // Start will run through all repos it has access to and check for updates and make pull requests if needed. func Start(ctx context.Context, client *github.Client, org string, repositories []string) { // If we are running in Bestseller specific mode, we need to set the running variable to true // To be able to query private orbs and docker images - config.AppConfig.BestsellerSpecific.Running = (org == "BESTSELLER") + config.AppConfig.BestsellerSpecific.Running = org == "BESTSELLER" // get repos // TODO: Get only repos in the list, but in a single API Call @@ -40,17 +50,64 @@ func Start(ctx context.Context, client *github.Client, org string, repositories } func checkRepo(ctx context.Context, client *github.Client, repo *github.Repository) { - // defer wg.Done() - repoName := repo.GetName() - - log.Debug().Msg(fmt.Sprintf("Checking repo: %s", repoName)) - // should we then remove the repo from our db ? if repo.GetArchived() { log.Debug().Msg(fmt.Sprintf("Repo '%s' is archived", repoName)) return } + var entityCount atomic.Int32 + + repoInfo, err := getRepoInfo(ctx, client, repo) + if err != nil { + log.Debug().Err(err).Msgf("could not get repo info for repo %s", repoName) + return + } + + go datadog.IncrementCount("analysed_repos", 1, []string{fmt.Sprintf("organization:%s", repoInfo.repoOwner)}) + updates := map[string]circleci.Update{} + var wg sync.WaitGroup + wg.Add(1) + go gatherUpdates(&wg, &entityCount, ctx, client, repoInfo, repoInfo.repoConfig.Directory, &updates) + wg.Wait() + + for newVerName, updateInfo := range updates { + oldBranch, prBranch, err := handleBranch(ctx, client, repoInfo, newVerName, updateInfo) + if err != nil || oldBranch { + return + } + prTitle := generatePRTitle(updateInfo, newVerName) + exists, oldPRs, err := gh.CheckPR(ctx, client, repoInfo.repoOwner, repoInfo.repoName, prTitle, updateInfo.CurrentName) + if err != nil { + log.Error().Err(err).Msgf("could not get old branch in %s", repoInfo.repoName) + return + } + if exists { + return + } + + err = handleUpdates(ctx, client, repoInfo, prBranch, &updateInfo) + if err != nil { + go datadog.IncrementCount("failed_repos", 1, []string{fmt.Sprintf("organization:%s", repoInfo.repoOwner)}) + return + } + prNumber, err := handlePR(ctx, client, repoInfo, prBranch, prTitle) + if err != nil { + return + } + if oldPRs != nil || len(oldPRs) > 0 { + gh.CleanUpOldBranch(ctx, client, repoInfo.repoOwner, repoInfo.repoName, oldPRs, prNumber) + go func() { + datadog.IncrementCount("superseeded_updates", 1, []string{fmt.Sprintf("organization:%s", repoInfo.repoOwner)}) + }() + } + } + +} + +func getRepoInfo(ctx context.Context, client *github.Client, repo *github.Repository) (*RepoInfo, error) { + repoName := repo.GetName() + log.Debug().Msg(fmt.Sprintf("Checking repo: %s", repoName)) // Use this to test the application against a single repo // if repoName != "bestone-bi4-sales-core-salesorderservice" { @@ -59,57 +116,105 @@ func checkRepo(ctx context.Context, client *github.Client, repo *github.Reposito repoConfig := getRepoConfig(ctx, client, repo) if repoConfig == nil { - return + return nil, errors.New(fmt.Sprintf("could not get repo config for repo %s", repoName)) } // determine repo details repoOwner := repo.GetOwner().GetLogin() repoDefaultBranch := repo.GetDefaultBranch() - targetBranch := getTargetBranch(ctx, client, repoOwner, repoName, repoDefaultBranch, repoConfig) if targetBranch == "" { - return + return nil, errors.New(fmt.Sprintf("could not get targetBranch for repo %s", repoName)) } + return &RepoInfo{ + repoConfig: repoConfig, + repoOwner: repoOwner, + repoDefaultBranch: repoDefaultBranch, + targetBranch: targetBranch, + repoName: repoName, + }, nil +} - go datadog.IncrementCount("analysed_repos", 1, []string{fmt.Sprintf("organization:%s", repoOwner)}) - - // get content of circleci config file - content, SHA, err := gh.GetRepoContent(ctx, client, repoOwner, repoName, repoConfig.Directory+"/.circleci/config.yml", targetBranch) +func handlePR(ctx context.Context, client *github.Client, info *RepoInfo, branchName, prTitle string) (int, error) { + // create pull req + newPR, err := gh.CreatePR(ctx, client, info.repoOwner, info.repoName, info.repoConfig.Reviewers, info.repoConfig.Assignees, info.repoConfig.Labels, &github.NewPullRequest{ + Title: github.String(prTitle), + Head: github.String(branchName), + Base: github.String(info.targetBranch), + Body: github.String(branchName), + MaintainerCanModify: github.Bool(true), + }) if err != nil { - return + log.Info().Err(err).Msgf("could not create pr in %s", info.repoName) + return -1, err } - // unmarshal - var cciconfig yaml.Node - err = yaml.Unmarshal(content, &cciconfig) - if err != nil { - log.Error().Err(err).Msgf("could not unmarshal yaml in %s", repoName) - return - } + go func() { + datadog.IncrementCount("pull_requests", 1, []string{fmt.Sprintf("organization:%s", info.repoOwner)}) + }() + return newPR.GetNumber(), nil +} - // check for updates - orbUpdates, dockerUpdates := circleci.GetUpdates(&cciconfig) - for old, update := range orbUpdates { - // wg.Add(1) - err = handleUpdate(ctx, client, update, "orb", old, content, repoOwner, repoName, targetBranch, SHA, repoConfig) +func handleBranch(ctx context.Context, client *github.Client, repoInfo *RepoInfo, newName string, updateInfo circleci.Update) (existed bool, branchName string, err error) { + oldVersion := updateInfo.SplitCurrentVersion() + commitBranch := fmt.Sprintf("dependabot-circleci/%s/%s@%s", updateInfo.Type, oldVersion[0], newName) + notExists := gh.CheckBranch(ctx, client, repoInfo.repoOwner, repoInfo.repoName, github.String(commitBranch)) + if notExists { + err = gh.CreateBranch(ctx, client, repoInfo.repoOwner, repoInfo.repoName, repoInfo.targetBranch, github.String(commitBranch)) if err != nil { - go datadog.IncrementCount("failed_repos", 1, []string{fmt.Sprintf("organization:%s", repoOwner)}) - return + log.Error().Err(err).Msgf("could not create branch %s in %s", commitBranch, repoInfo.repoName) + return false, "", err } + } else { + log.Debug().Msgf("branch %s already exists, skipping creation of branch", commitBranch) + return true, commitBranch, nil } - for old, update := range dockerUpdates { - // wg.Add(1) - err = handleUpdate(ctx, client, update, "docker", old, content, repoOwner, repoName, targetBranch, SHA, repoConfig) - if err != nil { - go datadog.IncrementCount("failed_repos", 1, []string{fmt.Sprintf("organization:%s", repoOwner)}) - return + return false, commitBranch, nil +} + +func gatherUpdates(wg *sync.WaitGroup, entityCount *atomic.Int32, ctx context.Context, client *github.Client, repoInfo *RepoInfo, pathInRepo string, updates *map[string]circleci.Update) { + defer wg.Done() + log.Info().Msgf("Processing: %s", pathInRepo) + // 1. Get directory contents + options := &github.RepositoryContentGetOptions{Ref: repoInfo.targetBranch} + fileContent, directoryContent, _, err := client.Repositories.GetContents(context.Background(), repoInfo.repoOwner, repoInfo.repoName, pathInRepo, options) + if err != nil { + log.Error().Err(err).Msgf("could not parseRepoContent %s", repoInfo.repoName) + return + } + if fileContent == nil { + for _, dir := range directoryContent { + if dir.GetType() == "dir" || isYaml(dir.GetName()) { + if entityCount.Load() > 100 { + log.Warn().Msgf("Repo with too many files: %s - %s", repoInfo.repoOwner, repoInfo.repoName) + return + } + entityCount.Add(1) + wg.Add(1) + go gatherUpdates(wg, entityCount, ctx, client, repoInfo, dir.GetPath(), updates) + } } + return + } + if !isYaml(fileContent.GetName()) { + log.Debug().Msgf("Skipping %s, not yml/yaml", fileContent.GetName()) + return + } + content, err := fileContent.GetContent() + if err != nil { + log.Error().Err(err).Msgf("could not fileContent.GetContent() %s", repoInfo.repoName) + } + // check for updates + err = circleci.ScanFileUpdates(updates, &content, &pathInRepo, fileContent.SHA) + if err != nil { + log.Warn().Err(err).Msgf("could not scan file updates in repo: %s, file:%s", repoInfo.repoName, pathInRepo) + return } } func getRepoConfig(ctx context.Context, client *github.Client, repo *github.Repository) *config.RepoConfig { // check if a bot config exists - repoConfigContent, _, err := gh.GetRepoContent(ctx, client, repo.GetOwner().GetLogin(), repo.GetName(), ".github/dependabot-circleci.yml", "") + repoConfigContent, _, err := gh.GetRepoFileBytes(ctx, client, repo.GetOwner().GetLogin(), repo.GetName(), ".github/dependabot-circleci.yml", "") if err != nil { log.Debug().Err(err).Msgf("could not load dependabot-circleci.yml in repo: %s", repo.GetName()) return nil @@ -136,85 +241,54 @@ func getTargetBranch(ctx context.Context, client *github.Client, repoOwner strin return targetBranch } -func handleUpdate(ctx context.Context, client *github.Client, update *yaml.Node, updateType string, old string, content []byte, repoOwner string, repoName string, targetBranch string, SHA *string, repoConfig *config.RepoConfig) error { - // defer wg.Done() - - log.Debug().Msgf("repo: %s, old: %s, update: %s", repoName, old, update.Value) - newYaml := circleci.ReplaceVersion(update, old, string(content)) - - // commit vars - var oldVersion, newVersion []string - if updateType == "orb" { - oldVersion = strings.Split(old, "@") - newVersion = strings.Split(update.Value, "@") - } else { - oldVersion = strings.Split(old, ":") - newVersion = strings.Split(update.Value, ":") - } - - if updateType == "orb" && len(newVersion) == 1 { - return fmt.Errorf("could not find orb version for %s in %s", update.Value, repoName) - } - - commitMessage := fmt.Sprintf("Bump @%s from %s to %s", oldVersion[0], oldVersion[1], newVersion[1]) - commitBranch := fmt.Sprintf("dependabot-circleci/%s/%s", updateType, strings.ReplaceAll(update.Value, ":", "@")) - - // err := check and create branch - exists, oldPRs, err := gh.CheckPR(ctx, client, repoOwner, repoName, targetBranch, commitBranch, commitMessage, oldVersion[0]) - if err != nil { - log.Error().Err(err).Msgf("could not get old branch in %s", repoName) - return err - } - if exists { - return err - } - - notExists := gh.CheckBranch(ctx, client, repoOwner, repoName, github.String(commitBranch)) - if notExists { - err = gh.CreateBranch(ctx, client, repoOwner, repoName, targetBranch, github.String(commitBranch)) +func handleUpdates(ctx context.Context, client *github.Client, info *RepoInfo, prBranch string, updates *circleci.Update) error { + for path, update := range updates.FileUpdates { + log.Debug().Msgf("repo: %s, file%s, old: %s, update: %s", info.repoName, path, updates.CurrentName, update.Node.Value) + // commit vars + oldVersion := updates.SplitCurrentVersion() + newVersion := update.Node.Value + param := circleci.ExtractParameterName(oldVersion[1]) + var newYaml string + if len(param) > 0 { + newYaml = circleci.ReplaceVersion((*update.Parameters)[param].Line-1, (*update.Parameters)[param].Value, newVersion, *update.Content) + } else { + newYaml = circleci.ReplaceVersion(update.Node.Line-1, oldVersion[1], newVersion, *update.Content) + } + commitMessage := fmt.Sprintf("Update %s, @%s to %s", path, oldVersion[0], newVersion) + + // commit file + err := gh.UpdateFile(ctx, client, info.repoOwner, info.repoName, path, &github.RepositoryContentFileOptions{ + Message: github.String(commitMessage), + Content: []byte(newYaml), + Branch: github.String(prBranch), + SHA: update.SHA, + }) if err != nil { - log.Error().Err(err).Msgf("could not create branch %s in %s", commitBranch, repoName) + log.Error().Err(err).Msgf("could not update file in %s", info.repoName) return err } - } else { - log.Debug().Msgf("branch %s already exists, skipping creation of branch", commitBranch) - } - - // commit file - err = gh.UpdateFile(ctx, client, repoOwner, repoName, repoConfig.Directory+"/.circleci/config.yml", &github.RepositoryContentFileOptions{ - Message: github.String(commitMessage), - Content: []byte(newYaml), - Branch: github.String(commitBranch), - SHA: SHA, - }) - if err != nil { - log.Error().Err(err).Msgf("could not update file in %s", repoName) - return err } + return nil +} - // create pull req - newPR, err := gh.CreatePR(ctx, client, repoOwner, repoName, repoConfig.Reviewers, repoConfig.Assignees, repoConfig.Labels, &github.NewPullRequest{ - Title: github.String(commitMessage), - Head: github.String(commitBranch), - Base: github.String(targetBranch), - Body: github.String(commitMessage), - MaintainerCanModify: github.Bool(true), - }) - if err != nil { - log.Info().Err(err).Msgf("could not create pr in %s", repoName) - return err +func generatePRTitle(update circleci.Update, newVersion string) string { + oldVersion := update.SplitCurrentVersion() + param := circleci.ExtractParameterName(oldVersion[1]) + oldVersionNumber := oldVersion[1] + if len(param) > 0 { + // Try getting version from first changed file + for _, v := range update.FileUpdates { + if oldParamValue, found := (*v.Parameters)[param]; found { + oldVersionNumber = oldParamValue.Value + } else { + oldVersionNumber = param + } + break + } } + return fmt.Sprintf("Bump @%s from %s to %s", oldVersion[0], oldVersionNumber, newVersion) +} - go func() { - datadog.IncrementCount("pull_requests", 1, []string{fmt.Sprintf("organization:%s", repoOwner)}) - }() - - if oldPRs != nil || len(oldPRs) > 0 { - gh.CleanUpOldBranch(ctx, client, repoOwner, repoName, oldPRs, newPR.GetNumber()) - - go func() { - datadog.IncrementCount("superseeded_updates", 1, []string{fmt.Sprintf("organization:%s", repoOwner)}) - }() - } - return nil +func isYaml(fileName string) bool { + return strings.HasSuffix(fileName, ".yml") || strings.HasSuffix(fileName, ".yaml") } diff --git a/gh/commit.go b/gh/commit.go index 92b664fb..043bc106 100644 --- a/gh/commit.go +++ b/gh/commit.go @@ -11,19 +11,18 @@ import ( ) // CheckPR . -func CheckPR(ctx context.Context, client *github.Client, repoOwner string, repoName string, baseBranch string, commitBranch string, commitMessage string, component string) (bool, []*github.PullRequest, error) { - PRsToBeClosed := []*github.PullRequest{} - pullreqs, _, _ := client.PullRequests.List(ctx, repoOwner, repoName, nil) - for _, pr := range pullreqs { - +func CheckPR(ctx context.Context, client *github.Client, repoOwner string, repoName string, expectedTitle string, component string) (bool, []*github.PullRequest, error) { + var PRsToBeClosed []*github.PullRequest + pullReqs, _, _ := client.PullRequests.List(ctx, repoOwner, repoName, nil) + for _, pr := range pullReqs { if pr.GetUser().GetLogin() != "dependabot-circleci[bot]" { continue } title := pr.GetTitle() - // exists ? - if title == commitMessage { + if title == expectedTitle { + log.Debug().Str("repo_name", repoName).Str("pr_title", title).Msg("PR already exists") return true, nil, nil } @@ -31,12 +30,10 @@ func CheckPR(ctx context.Context, client *github.Client, repoOwner string, repoN if strings.Contains(title, fmt.Sprintf("@%s", component)) { PRsToBeClosed = append(PRsToBeClosed, pr) } - } if len(PRsToBeClosed) > 0 { return false, PRsToBeClosed, nil } - return false, nil, nil } diff --git a/gh/repos.go b/gh/repos.go index 0a70b65c..c189b5c3 100644 --- a/gh/repos.go +++ b/gh/repos.go @@ -24,14 +24,13 @@ func GetRepos(ctx context.Context, client *github.Client, page int) ([]*github.R return repos, nil } -// GetRepoContent returns the circleci config as a byte array -func GetRepoContent(ctx context.Context, client *github.Client, owner string, repo string, file string, branch string) ([]byte, *string, error) { +// GetRepoFileBytes returns the circleci config as a byte array +func GetRepoFileBytes(ctx context.Context, client *github.Client, owner string, repo string, file string, branch string) ([]byte, *string, error) { options := &github.RepositoryContentGetOptions{Ref: branch} fileContent, _, _, err := client.Repositories.GetContents(context.Background(), owner, repo, file, options) if err != nil { return nil, nil, err } - content, err := fileContent.GetContent() if err != nil { return nil, nil, err diff --git a/go.mod b/go.mod index 1108c53e..312e77b8 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/philhofer/fwd v1.1.2 // indirect @@ -106,6 +107,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect + gotest.tools/v3 v3.5.1 // indirect mellium.im/sasl v0.3.1 // indirect ) diff --git a/go.sum b/go.sum index 6f1eac4a..2c11676d 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= github.com/palantir/go-baseapp v0.5.2 h1:b1ukx7AXo2/E4NkUvTFlW+185uwCcifzd2XzLrG4oS8= @@ -395,8 +395,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/gotraceui v0.2.0 h1:dmNsfQ9Vl3GwbiVD7Z8d/osC6WtGGrasyrC2suc4ZIQ= honnef.co/go/gotraceui v0.2.0/go.mod h1:qHo4/W75cA3bX0QQoSvDjbJa4R8mAyyFjbWAj63XElc= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 8b88c102..dd3c9704 100644 --- a/main.go +++ b/main.go @@ -39,12 +39,12 @@ func init() { } dbsecret = bytes - err = config.ReadAppConfig([]byte(appsecret)) + err = config.ReadAppConfig(appsecret) if err != nil { log.Fatal().Err(err).Msg("failed to read github app config:") } - err = config.ReadDBConfig([]byte(dbsecret)) + err = config.ReadDBConfig(dbsecret) if err != nil { log.Fatal().Err(err).Msg("failed to read db config:") }