From d46c0d2e361423ea019bc2d18049066c889ac14e Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Thu, 17 Feb 2022 07:45:10 -0800 Subject: [PATCH] Add dependabot-generate make target (#2613) * Refactor common repo code for crosslink * Add dbotconf utility * Add dependabot-generate target to Makefile * Generate dependabot.yml * Update Makefile targets related to dependabot-generate --- .github/dependabot.yml | 170 +++++++++----------------- Makefile | 13 +- internal/tools/common.go | 79 +++++++++++- internal/tools/crosslink/crosslink.go | 112 ++++------------- internal/tools/dbotconf/dbotconf.go | 112 +++++++++++++++++ internal/tools/go.mod | 1 + 6 files changed, 276 insertions(+), 211 deletions(-) create mode 100644 internal/tools/dbotconf/dbotconf.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d326b38e49ca..ca2daee0cbdd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,8 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates -# \todo Eliminate duplication when/if Dependabot supports YAML anchors +# File generated by "make dependabot-generate"; DO NOT EDIT. version: 2 updates: - - - package-ecosystem: github-actions + - package-ecosystem: github-actions directory: / labels: - dependencies @@ -16,8 +11,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: / labels: - dependencies @@ -26,18 +20,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /bridge/opentracing - labels: - - dependencies - - go - - "Skip Changelog" - schedule: - day: sunday - interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /bridge/opencensus labels: - dependencies @@ -46,8 +29,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /bridge/opencensus/test labels: - dependencies @@ -56,9 +38,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /example/fib + - package-ecosystem: gomod + directory: /bridge/opentracing labels: - dependencies - go @@ -66,9 +47,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /example/prom-collector + - package-ecosystem: gomod + directory: /example/fib labels: - dependencies - go @@ -76,8 +56,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/jaeger labels: - dependencies @@ -86,8 +65,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/namedtracer labels: - dependencies @@ -96,8 +74,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/opencensus labels: - dependencies @@ -106,8 +83,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/otel-collector labels: - dependencies @@ -116,8 +92,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/passthrough labels: - dependencies @@ -126,8 +101,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/prometheus labels: - dependencies @@ -136,8 +110,7 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod + - package-ecosystem: gomod directory: /example/zipkin labels: - dependencies @@ -146,19 +119,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/prometheus - labels: - - dependencies - - go - - "Skip Changelog" - schedule: - day: sunday - interval: weekly - - - package-ecosystem: gomod - directory: /exporters/stdout/stdouttrace + - package-ecosystem: gomod + directory: /exporters/jaeger labels: - dependencies - go @@ -166,9 +128,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/stdout/stdoutmetric + - package-ecosystem: gomod + directory: /exporters/otlp/internal/retry labels: - dependencies - go @@ -176,9 +137,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/jaeger + - package-ecosystem: gomod + directory: /exporters/otlp/otlpmetric labels: - dependencies - go @@ -186,9 +146,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/zipkin + - package-ecosystem: gomod + directory: /exporters/otlp/otlpmetric/otlpmetricgrpc labels: - dependencies - go @@ -196,9 +155,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /sdk + - package-ecosystem: gomod + directory: /exporters/otlp/otlpmetric/otlpmetrichttp labels: - dependencies - go @@ -206,9 +164,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /internal/metric + - package-ecosystem: gomod + directory: /exporters/otlp/otlptrace labels: - dependencies - go @@ -216,9 +173,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /internal/tools + - package-ecosystem: gomod + directory: /exporters/otlp/otlptrace/otlptracegrpc labels: - dependencies - go @@ -226,9 +182,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /metric + - package-ecosystem: gomod + directory: /exporters/otlp/otlptrace/otlptracehttp labels: - dependencies - go @@ -236,9 +191,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /sdk/export/metric + - package-ecosystem: gomod + directory: /exporters/prometheus labels: - dependencies - go @@ -246,9 +200,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /sdk/metric + - package-ecosystem: gomod + directory: /exporters/stdout/stdoutmetric labels: - dependencies - go @@ -256,9 +209,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /internal/tools/semconv-gen + - package-ecosystem: gomod + directory: /exporters/stdout/stdouttrace labels: - dependencies - go @@ -266,9 +218,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/internal/retry + - package-ecosystem: gomod + directory: /exporters/zipkin labels: - dependencies - go @@ -276,9 +227,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/otlptrace + - package-ecosystem: gomod + directory: /internal/metric labels: - dependencies - go @@ -286,9 +236,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/otlptrace/otlptracegrpc + - package-ecosystem: gomod + directory: /internal/tools labels: - dependencies - go @@ -296,9 +245,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/otlptrace/otlptracehttp + - package-ecosystem: gomod + directory: /metric labels: - dependencies - go @@ -306,9 +254,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/otlpmetric + - package-ecosystem: gomod + directory: /schema labels: - dependencies - go @@ -316,9 +263,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/otlpmetric/otlpmetricgrpc + - package-ecosystem: gomod + directory: /sdk labels: - dependencies - go @@ -326,9 +272,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /trace + - package-ecosystem: gomod + directory: /sdk/export/metric labels: - dependencies - go @@ -336,9 +281,8 @@ updates: schedule: day: sunday interval: weekly - - - package-ecosystem: gomod - directory: /exporters/otlp/otlpmetric/otlpmetrichttp + - package-ecosystem: gomod + directory: /sdk/metric labels: - dependencies - go @@ -346,10 +290,8 @@ updates: schedule: day: sunday interval: weekly - - - - package-ecosystem: gomod - directory: /schema + - package-ecosystem: gomod + directory: /trace labels: - dependencies - go diff --git a/Makefile b/Makefile index b085561dbaaf..bcb3a6d86295 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ TIMEOUT = 60 .DEFAULT_GOAL := precommit .PHONY: precommit ci -precommit: license-check misspell go-mod-tidy golangci-lint-fix test-default +precommit: dependabot-generate license-check misspell go-mod-tidy golangci-lint-fix test-default ci: dependabot-check license-check lint vanity-import-check build test-default check-clean-work-tree test-coverage # Tools @@ -47,6 +47,9 @@ $(TOOLS)/semconvgen: PACKAGE=go.opentelemetry.io/build-tools/semconvgen CROSSLINK = $(TOOLS)/crosslink $(TOOLS)/crosslink: PACKAGE=go.opentelemetry.io/otel/$(TOOLS_MOD_DIR)/crosslink +DBOTCONF = $(TOOLS)/dbotconf +$(TOOLS)/dbotconf: PACKAGE=go.opentelemetry.io/otel/$(TOOLS_MOD_DIR)/dbotconf + GOLANGCI_LINT = $(TOOLS)/golangci-lint $(TOOLS)/golangci-lint: PACKAGE=github.com/golangci/golangci-lint/cmd/golangci-lint @@ -66,7 +69,7 @@ GOJQ = $(TOOLS)/gojq $(TOOLS)/gojq: PACKAGE=github.com/itchyny/gojq/cmd/gojq .PHONY: tools -tools: $(CROSSLINK) $(GOLANGCI_LINT) $(MISSPELL) $(GOCOVMERGE) $(STRINGER) $(PORTO) $(GOJQ) $(SEMCONVGEN) $(MULTIMOD) +tools: $(CROSSLINK) $(DBOTCONF) $(GOLANGCI_LINT) $(MISSPELL) $(GOCOVMERGE) $(STRINGER) $(PORTO) $(GOJQ) $(SEMCONVGEN) $(MULTIMOD) # Build @@ -187,9 +190,15 @@ dependabot-check: if [ -n "$$result" ]; then \ echo "missing dependabot entry:"; echo "$$result"; \ echo "new modules need to be added to the $(DEPENDABOT_PATH) file"; \ + echo "(run: make dependabot-generate)"; \ exit 1; \ fi +.PHONY: dependabot-generate +dependabot-generate: $(DBOTCONF) + @echo "gerating dependabot configuration"; \ + $(DBOTCONF) + .PHONY: check-clean-work-tree check-clean-work-tree: @if ! git diff --quiet; then \ diff --git a/internal/tools/common.go b/internal/tools/common.go index 525e5adcf3be..0c7389934ca2 100644 --- a/internal/tools/common.go +++ b/internal/tools/common.go @@ -18,18 +18,28 @@ package tools // import "go.opentelemetry.io/otel/internal/tools" import ( + "bytes" "errors" "fmt" + "io" "os" "path/filepath" + "sort" "strings" + "text/tabwriter" + + "golang.org/x/mod/modfile" ) -// FindRepoRoot retrieves the root of the repository containing the current working directory. -// Beginning at the current working directory (dir), the algorithm checks if joining the ".git" -// suffix, such as "dir.get", is a valid file. Otherwise, it will continue checking the dir's -// parent directory until it reaches the repo root or returns an error if it cannot be found. -func FindRepoRoot() (string, error) { +// Repo represents a git repository. +type Repo string + +// FindRepoRoot retrieves the root of the repository containing the current +// working directory. Beginning at the current working directory (dir), the +// algorithm checks if joining the ".git" suffix, such as "dir.get", is a +// valid file. Otherwise, it will continue checking the dir's parent directory +// until it reaches the repo root or returns an error if it cannot be found. +func FindRepoRoot() (Repo, error) { start, err := os.Getwd() if err != nil { return "", err @@ -52,6 +62,63 @@ func FindRepoRoot() (string, error) { return "", err } - return dir, nil + return Repo(dir), nil + } +} + +// FindModules returns all Go modules contained in Repo r. +func (r Repo) FindModules() ([]*modfile.File, error) { + var results []*modfile.File + err := filepath.Walk(string(r), func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + // Walk failed to walk into this directory. Stop walking and + // signal this error. + return walkErr + } + + if !info.IsDir() { + return nil + } + + goMod := filepath.Join(path, "go.mod") + f, err := os.Open(goMod) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + + var b bytes.Buffer + io.Copy(&b, f) + if err = f.Close(); err != nil { + return err + } + + mFile, err := modfile.Parse(goMod, b.Bytes(), nil) + if err != nil { + return err + } + results = append(results, mFile) + return nil + }) + + sort.SliceStable(results, func(i, j int) bool { + return results[i].Syntax.Name < results[j].Syntax.Name + }) + + return results, err +} + +func PrintModFiles(w io.Writer, mFiles []*modfile.File) error { + tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) + if _, err := fmt.Fprintln(tw, "FILE PATH\tIMPORT PATH"); err != nil { + return err + } + for _, m := range mFiles { + if _, err := fmt.Fprintf(tw, "%s\t%s\n", m.Syntax.Name, m.Module.Mod.Path); err != nil { + return err + } } + return tw.Flush() } diff --git a/internal/tools/crosslink/crosslink.go b/internal/tools/crosslink/crosslink.go index a229bfa6db01..5f19d5b946ed 100644 --- a/internal/tools/crosslink/crosslink.go +++ b/internal/tools/crosslink/crosslink.go @@ -25,106 +25,42 @@ package main import ( - "encoding/json" - "errors" - "fmt" - "io" "log" "os" - "os/exec" "path/filepath" "strings" - "text/tabwriter" "go.opentelemetry.io/otel/internal/tools" + "golang.org/x/mod/modfile" ) -type repo string - -type mod struct { - filePath string - importPath string -} - -func (r repo) findModules() (mods, error) { - var results []mod - err := filepath.Walk(string(r), func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - return nil - } - - _, err = os.Stat(filepath.Join(path, "go.mod")) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return err - } - - cmd := exec.Command("go", "mod", "edit", "-json") - cmd.Dir = path - out, err := cmd.Output() - if err != nil { - return err - } - - var result struct { - Module struct { - Path string - } - } - err = json.Unmarshal(out, &result) - if err != nil { - return err - } - - results = append(results, mod{ - filePath: path, - importPath: result.Module.Path, - }) - return nil - }) - - return results, err -} - -type mods []mod - -func (m mods) print(w io.Writer) error { - tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) - if _, err := fmt.Fprintln(tw, "FILE PATH\tIMPORT PATH"); err != nil { - return err - } - for _, m := range m { - if _, err := fmt.Fprintf(tw, "%s\t%s\n", m.filePath, m.importPath); err != nil { - return err - } - } - return tw.Flush() -} - -func (m mods) crossLink() error { +func crossLink(m []*modfile.File) error { for _, from := range m { - args := []string{"mod", "edit"} - + basepath := filepath.Dir(from.Syntax.Name) for _, to := range m { - localPath, err := filepath.Rel(from.filePath, to.filePath) + newPath, err := filepath.Rel(basepath, filepath.Dir(to.Syntax.Name)) if err != nil { return err } - if localPath == "." || localPath == ".." { - localPath += "/" - } else if !strings.HasPrefix(localPath, "..") { - localPath = "./" + localPath + switch { + case newPath == ".", newPath == "..": + newPath += "/" + case !strings.HasPrefix(newPath, ".."): + newPath = "./" + newPath } - args = append(args, "-replace", to.importPath+"="+localPath) + from.AddReplace(to.Module.Mod.Path, "", newPath, "") } - cmd := exec.Command("go", args...) - cmd.Dir = from.filePath - out, err := cmd.CombinedOutput() + from.Cleanup() + + f, err := os.OpenFile(from.Syntax.Name, os.O_RDWR|os.O_TRUNC, 0755) if err != nil { - log.Println(string(out)) + return err + } + if _, err = f.Write(modfile.Format(from.Syntax)); err != nil { + return err + } + if err = f.Close(); err != nil { return err } } @@ -132,23 +68,21 @@ func (m mods) crossLink() error { } func main() { - repoRootStr, err := tools.FindRepoRoot() + root, err := tools.FindRepoRoot() if err != nil { log.Fatalf("unable to find repo root: %v", err) } - repoRoot := repo(repoRootStr) - - mods, err := repoRoot.findModules() + mods, err := root.FindModules() if err != nil { log.Fatalf("unable to list modules: %v", err) } - if err := mods.print(os.Stdout); err != nil { + if err := tools.PrintModFiles(os.Stdout, mods); err != nil { log.Fatalf("unable to print modules: %v", err) } - if err := mods.crossLink(); err != nil { + if err := crossLink(mods); err != nil { log.Fatalf("unable to crosslink: %v", err) } } diff --git a/internal/tools/dbotconf/dbotconf.go b/internal/tools/dbotconf/dbotconf.go new file mode 100644 index 000000000000..a81a3a83fb99 --- /dev/null +++ b/internal/tools/dbotconf/dbotconf.go @@ -0,0 +1,112 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main provides a utility to generate a complete dependabot +// configuration for a repository with multiple Go modules. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + "text/template" + + "go.opentelemetry.io/otel/internal/tools" + "golang.org/x/mod/modfile" +) + +var configPtr = flag.String("config", "./.github/dependabot.yml", "dependabot configuration path") + +const configTemplate = `# File generated by "make dependabot-generate"; DO NOT EDIT. + +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + labels: + - dependencies + - actions + - "Skip Changelog" + schedule: + day: sunday + interval: weekly +{{- range .}} + - package-ecosystem: gomod + directory: {{.}} + labels: + - dependencies + - go + - "Skip Changelog" + schedule: + day: sunday + interval: weekly +{{- end}} +` + +func gomodDirectories(basePath string, mods []*modfile.File) []string { + var dirs []string + for _, m := range mods { + targetPath := filepath.Dir(m.Syntax.Name) + relPath := strings.TrimPrefix(targetPath, basePath) + if relPath == "" { + relPath = "/" + } + dirs = append(dirs, relPath) + } + sort.Strings(dirs) + return dirs +} + +func generate(path string) error { + tpl, err := template.New("dependabot.yml").Parse(configTemplate) + if err != nil { + return fmt.Errorf("parse template: %w", err) + } + + root, err := tools.FindRepoRoot() + if err != nil { + return fmt.Errorf("find repo root: %w", err) + } + + mods, err := root.FindModules() + if err != nil { + return fmt.Errorf("list modules: %w", err) + } + data := gomodDirectories(string(root), mods) + + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return err + } + if err = tpl.Execute(f, data); err != nil { + // Best effort. + _ = f.Close() + return fmt.Errorf("rendering template: %w", err) + } + if err = f.Close(); err != nil { + return fmt.Errorf("closing %s: %w", path, err) + } + return nil +} + +func main() { + flag.Parse() + if err := generate(*configPtr); err != nil { + log.Fatalf("failed to generate dependabot configuration: %v", err) + } +} diff --git a/internal/tools/go.mod b/internal/tools/go.mod index fbaac2e35a90..aac8b9cb1c04 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -11,6 +11,7 @@ require ( github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad go.opentelemetry.io/build-tools/multimod v0.0.0-20210920164323-2ceabab23375 go.opentelemetry.io/build-tools/semconvgen v0.0.0-20210920164323-2ceabab23375 + golang.org/x/mod v0.5.1 golang.org/x/tools v0.1.9 )