diff --git a/README.md b/README.md index 3351754db1..d6dce365e7 100644 --- a/README.md +++ b/README.md @@ -237,16 +237,5 @@ only applies to the line on which it appears. - uses: 'actions/checkout@v${{ matrix.version }}' ``` -## Experiments - -Use these options to configure the default behavior of Ratchet. - -### Experiment: Keep Newlines - -Experimental functionality to enable keeping newlines in the output. This only -applies to cli commands that modify output. As of v0.5.0, this functionality is -enabled by default. To disable it, set the environment variable -`RATCHET_EXP_KEEP_NEWLINES=false`. - [containers]: https://github.com/sethvargo/ratchet/pkgs/container/ratchet [releases]: https://github.com/sethvargo/ratchet/releases diff --git a/command/check.go b/command/check.go index 540f10455a..e41f1aa81b 100644 --- a/command/check.go +++ b/command/check.go @@ -13,6 +13,8 @@ import ( const checkCommandDesc = `Check if all versions are pinned` const checkCommandHelp = ` +Usage: ratchet check [FILE...] + The "check" command checks if all versions are pinned to an absolute version, ignoring any versions with the "ratchet:exclude" comment. @@ -48,21 +50,9 @@ func (c *CheckCommand) Flags() *flag.FlagSet { } func (c *CheckCommand) Run(ctx context.Context, originalArgs []string) error { - f := c.Flags() - - if err := f.Parse(originalArgs); err != nil { - return fmt.Errorf("failed to parse flags: %w", err) - } - - args := f.Args() - if got := len(args); got != 1 { - return fmt.Errorf("expected exactly one argument, got %d", got) - } - - inFile := args[0] - m, err := parseYAMLFile(inFile) + args, err := parseFlags(c.Flags(), originalArgs) if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) + return fmt.Errorf("failed to parse flags: %w", err) } par, err := parser.For(ctx, c.flagParser) @@ -70,9 +60,12 @@ func (c *CheckCommand) Run(ctx context.Context, originalArgs []string) error { return err } - if err := parser.Check(ctx, par, m); err != nil { - return fmt.Errorf("check failed: %w", err) + fsys := os.DirFS(".") + + files, err := loadYAMLFiles(fsys, args) + if err != nil { + return err } - return nil + return parser.Check(ctx, par, files.nodes()) } diff --git a/command/cmd/gen/main.go b/command/cmd/gen/main.go index ad7d20149f..ede9087a33 100644 --- a/command/cmd/gen/main.go +++ b/command/cmd/gen/main.go @@ -38,7 +38,7 @@ func realMain() error { root := folderRoot() pth := path.Join(root, "command_gen.go") - if err := os.WriteFile(pth, w.Bytes(), 0o644); err != nil { + if err := os.WriteFile(pth, w.Bytes(), 0o600); err != nil { return fmt.Errorf("failed to write generated file: %w", err) } diff --git a/command/command.go b/command/command.go index c68c9d24c2..3b02093603 100644 --- a/command/command.go +++ b/command/command.go @@ -2,11 +2,13 @@ package command import ( + "bytes" "context" + "errors" + "flag" "fmt" - "io" + "io/fs" "os" - "strconv" "strings" // Using banydonk/yaml instead of the default yaml pkg because the default @@ -15,8 +17,6 @@ import ( "github.com/hexops/gotextdiff" "github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/span" - - "github.com/sethvargo/ratchet/internal/atomic" "github.com/sethvargo/ratchet/internal/version" ) @@ -57,14 +57,26 @@ func Run(ctx context.Context, args []string) error { return cmd.Run(ctx, args) } -func keepNewlinesEnv() bool { - value := true - if v, ok := os.LookupEnv("RATCHET_EXP_KEEP_NEWLINES"); ok { - if t, err := strconv.ParseBool(v); err == nil { - value = t +// parseFlags is a helper that parses flags. Unlike [flags.Parse], it handles +// flags that occur after or between positional arguments. +func parseFlags(f *flag.FlagSet, args []string) ([]string, error) { + var finalArgs []string + var merr error + + merr = errors.Join(merr, f.Parse(args)) + + for i := len(args) - len(f.Args()) + 1; i < len(args); { + // Stop parsing if we hit an actual "stop parsing" + if i > 1 && args[i-2] == "--" { + break } + finalArgs = append(finalArgs, f.Arg(0)) + merr = errors.Join(merr, f.Parse(args[i:])) + i += 1 + len(args[i:]) - len(f.Args()) } - return value + finalArgs = append(finalArgs, f.Args()...) + + return finalArgs, merr } // extractCommandAndArgs is a helper that pulls the subcommand and arguments. @@ -79,106 +91,59 @@ func extractCommandAndArgs(args []string) (string, []string) { } } -// writeYAML encodes the yaml node into the given writer. -func writeYAML(w io.Writer, m *yaml.Node) error { - enc := yaml.NewEncoder(w) +// marshalYAML encodes the yaml node into the given writer. +func marshalYAML(m *yaml.Node) ([]byte, error) { + var b bytes.Buffer + + enc := yaml.NewEncoder(&b) enc.SetIndent(2) if err := enc.Encode(m); err != nil { - return fmt.Errorf("failed to encode yaml: %w", err) + return nil, fmt.Errorf("failed to encode yaml: %w", err) } if err := enc.Close(); err != nil { - return fmt.Errorf("failed to finalize yaml: %w", err) + return nil, fmt.Errorf("failed to finalize yaml: %w", err) } - return nil + return b.Bytes(), nil } -// writeYAMLFile renders the given yaml and atomically writes it to the provided -// filepath. -func writeYAMLFile(src, dst string, m *yaml.Node) (retErr error) { - r, w := io.Pipe() - defer func() { - if err := r.Close(); err != nil && retErr == nil { - retErr = fmt.Errorf("failed to close reader: %w", err) - } - }() - defer func() { - if err := w.Close(); err != nil && retErr == nil { - retErr = fmt.Errorf("failed to closer writer: %w", err) - } - }() - - errCh := make(chan error, 1) - go func() { - if err := writeYAML(w, m); err != nil { - select { - case errCh <- fmt.Errorf("failed to render yaml: %w", err): - default: - } - } - - if err := w.Close(); err != nil { - select { - case errCh <- fmt.Errorf("failed to close writer: %w", err): - default: - } - } - }() +type loadResult struct { + path string + node *yaml.Node + contents []byte +} - if err := atomic.Write(src, dst, r); err != nil { - retErr = fmt.Errorf("failed to save file %s: %w", dst, err) - return - } +type loadResults []*loadResult - select { - case err := <-errCh: - retErr = err - return - default: - return +func (r loadResults) nodes() []*yaml.Node { + n := make([]*yaml.Node, 0, len(r)) + for _, v := range r { + n = append(n, v.node) } + return n } -// parseYAML parses the given reader as a yaml node. -func parseYAML(r io.Reader) (*yaml.Node, error) { - var m yaml.Node - if err := yaml.NewDecoder(r).Decode(&m); err != nil { - return nil, fmt.Errorf("failed to decode yaml: %w", err) - } - return &m, nil -} +func loadYAMLFiles(fsys fs.FS, paths []string) (loadResults, error) { + r := make(loadResults, 0, len(paths)) -// parseYAMLFile opens the file at the path and parses it as yaml. It closes the -// file handle. -func parseYAMLFile(pth string) (m *yaml.Node, retErr error) { - f, err := os.Open(pth) - if err != nil { - retErr = fmt.Errorf("failed to open file: %w", err) - return - } - defer func() { - if err := f.Close(); err != nil && retErr == nil { - retErr = fmt.Errorf("failed to close file: %w", err) + for _, pth := range paths { + contents, err := fs.ReadFile(fsys, pth) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", pth, err) } - }() - m, retErr = parseYAML(f) - return -} + var node yaml.Node + if err := yaml.Unmarshal(contents, &node); err != nil { + return nil, fmt.Errorf("failed to parse yaml for %s: %w", pth, err) + } -func parseFile(pth string) (contents string, retErr error) { - f, err := os.Open(pth) - if err != nil { - retErr = fmt.Errorf("failed to open file: %w", err) - return + r = append(r, &loadResult{ + path: pth, + node: &node, + contents: contents, + }) } - defer func() { - if err := f.Close(); err != nil && retErr == nil { - retErr = fmt.Errorf("failed to close file: %w", err) - } - }() - c, retErr := io.ReadAll(f) - contents = string(c) - return + + return r, nil } func removeNewLineChanges(beforeContent, afterContent string) string { diff --git a/command/command_test.go b/command/command_test.go index de6f155122..ab65c6a575 100644 --- a/command/command_test.go +++ b/command/command_test.go @@ -2,14 +2,10 @@ package command import ( "bytes" - "fmt" - "path/filepath" + "os" "reflect" - "runtime" "testing" - // Using banydonk/yaml instead of the default yaml pkg because the default - // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" "github.com/google/go-cmp/cmp" ) @@ -232,7 +228,7 @@ jobs: it has many lines some of them even - have new new lines + have new lines ` yamlDChanges = ` jobs: @@ -253,7 +249,7 @@ jobs: it has many lines some of them even - have new new lines + have new lines ` yamlDChangesFormatted = ` jobs: @@ -275,7 +271,7 @@ jobs: it has many lines some of them even - have new new lines + have new lines ` ) @@ -329,17 +325,17 @@ func Test_removeNewLineChanges(t *testing.T) { } } -func Test_parseYAMLFile(t *testing.T) { +func Test_loadYAMLFiles(t *testing.T) { t.Parallel() cases := []struct { - name string - yamlFilename string - want string + name string + yamlFilenames []string + want string }{ { - name: "yamlA_multiple_empty_lines", - yamlFilename: "testdata/github.yml", + name: "yamlA_multiple_empty_lines", + yamlFilenames: []string{"testdata/github.yml"}, want: `jobs: my_job: runs-on: 'ubuntu-latest' @@ -370,26 +366,20 @@ func Test_parseYAMLFile(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - node, err := parseYAMLFile(rootPath(tc.yamlFilename)) + files, err := loadYAMLFiles(os.DirFS(".."), tc.yamlFilenames) if err != nil { - t.Errorf("parseYAMLFile() returned error: %v", err) + t.Fatalf("loadYAMLFiles() returned error: %s", err) } + var buf bytes.Buffer - err = yaml.NewEncoder(&buf).Encode(node) - if err != nil { - t.Errorf("failed to marshal yaml to string: %v", err) + if err := yaml.NewEncoder(&buf).Encode(files.nodes()[0]); err != nil { + t.Errorf("failed to marshal yaml to string: %s", err) } got := buf.String() if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("removeBindingFromPolicy() returned diff (-want +got):\n%s", diff) + t.Errorf("returned diff (-want, +got):\n%s", diff) } }) } } - -func rootPath(filename string) (fn string) { - _, fn, _, _ = runtime.Caller(0) - repoRoot := filepath.Dir(filepath.Dir(fn)) - return fmt.Sprintf("%s/%s", repoRoot, filename) -} diff --git a/command/pin.go b/command/pin.go index fc2f3052f8..cb7e7980b8 100644 --- a/command/pin.go +++ b/command/pin.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "strings" "github.com/sethvargo/ratchet/internal/atomic" @@ -16,6 +17,8 @@ import ( const pinCommandDesc = `Resolve and pin all versions` const pinCommandHelp = ` +Usage: ratchet pin [FILE...] + The "pin" command resolves and pins any unpinned versions to their absolute or hashed version for the given input file: @@ -60,27 +63,9 @@ func (c *PinCommand) Flags() *flag.FlagSet { } func (c *PinCommand) Run(ctx context.Context, originalArgs []string) error { - f := c.Flags() - - if err := f.Parse(originalArgs); err != nil { - return fmt.Errorf("failed to parse flags: %w", err) - } - - args := f.Args() - if got := len(args); got != 1 { - return fmt.Errorf("expected exactly one argument, got %d", got) - } - - inFile := args[0] - - uneditedContent, err := parseFile(inFile) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) - } - - m, err := parseYAMLFile(inFile) + args, err := parseFlags(c.Flags(), originalArgs) if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) + return fmt.Errorf("failed to parse flags: %w", err) } par, err := parser.For(ctx, c.flagParser) @@ -90,34 +75,42 @@ func (c *PinCommand) Run(ctx context.Context, originalArgs []string) error { res, err := resolver.NewDefaultResolver(ctx) if err != nil { - return fmt.Errorf("failed to create github resolver: %w", err) - } - - if err := parser.Pin(ctx, res, par, m, c.flagConcurrency); err != nil { - return fmt.Errorf("failed to pin refs: %w", err) + return fmt.Errorf("failed to create resolver: %w", err) } - outFile := c.flagOut - if outFile == "" { - outFile = inFile - } + fsys := os.DirFS(".") - if err := writeYAMLFile(inFile, outFile, m); err != nil { - return fmt.Errorf("failed to save %s: %w", outFile, err) + files, err := loadYAMLFiles(fsys, args) + if err != nil { + return err } - if !keepNewlinesEnv() { - return nil + if len(files) > 1 && c.flagOut != "" && !strings.HasSuffix(c.flagOut, "/") { + return fmt.Errorf("-out must be a directory when pinning multiple files") } - editedContent, err := parseFile(outFile) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", outFile, err) + if err := parser.Pin(ctx, res, par, files.nodes(), c.flagConcurrency); err != nil { + return fmt.Errorf("failed to pin refs: %w", err) } - final := removeNewLineChanges(uneditedContent, editedContent) - if err := atomic.Write(inFile, outFile, strings.NewReader(final)); err != nil { - return fmt.Errorf("failed to save file %s: %w", outFile, err) + for _, f := range files { + outFile := c.flagOut + if strings.HasSuffix(c.flagOut, "/") { + outFile = filepath.Join(c.flagOut, f.path) + } + if outFile == "" { + outFile = f.path + } + + updated, err := marshalYAML(f.node) + if err != nil { + return fmt.Errorf("failed to marshal yaml for %s: %w", f.path, err) + } + + final := removeNewLineChanges(string(f.contents), string(updated)) + if err := atomic.Write(f.path, outFile, strings.NewReader(final)); err != nil { + return fmt.Errorf("failed to save file %s: %w", outFile, err) + } } return nil diff --git a/command/unpin.go b/command/unpin.go index 426e0d676d..c09e3fb427 100644 --- a/command/unpin.go +++ b/command/unpin.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "strings" "github.com/sethvargo/ratchet/internal/atomic" @@ -14,6 +15,8 @@ import ( const unpinCommandDesc = `Revert pinned versions to their unpinned values` const unpinCommandHelp = ` +Usage: ratchet unpin [FILE...] + The "unpin" command reverts any pinned versions to their non-absolute or relative version for the given input file: @@ -54,52 +57,44 @@ func (c *UnpinCommand) Flags() *flag.FlagSet { } func (c *UnpinCommand) Run(ctx context.Context, originalArgs []string) error { - f := c.Flags() - - if err := f.Parse(originalArgs); err != nil { + args, err := parseFlags(c.Flags(), originalArgs) + if err != nil { return fmt.Errorf("failed to parse flags: %w", err) } - args := f.Args() - if got := len(args); got != 1 { - return fmt.Errorf("expected exactly one argument, got %d", got) - } - - inFile := args[0] - uneditedContent, err := parseFile(inFile) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) - } + fsys := os.DirFS(".") - m, err := parseYAMLFile(inFile) + files, err := loadYAMLFiles(fsys, args) if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) + return err } - if err := parser.Unpin(m); err != nil { - return fmt.Errorf("failed to upin refs: %w", err) + if len(files) > 1 && c.flagOut != "" && !strings.HasSuffix(c.flagOut, "/") { + return fmt.Errorf("-out must be a directory when pinning multiple files") } - outFile := c.flagOut - if outFile == "" { - outFile = inFile - } - if err := writeYAMLFile(inFile, outFile, m); err != nil { - return fmt.Errorf("failed to save %s: %w", outFile, err) - } - - if !keepNewlinesEnv() { - return nil - } - - editedContent, err := parseFile(outFile) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", outFile, err) + if err := parser.Unpin(ctx, files.nodes()); err != nil { + return fmt.Errorf("failed to pin refs: %w", err) } - final := removeNewLineChanges(uneditedContent, editedContent) - if err := atomic.Write(inFile, outFile, strings.NewReader(final)); err != nil { - return fmt.Errorf("failed to save file %s: %w", outFile, err) + for _, f := range files { + outFile := c.flagOut + if strings.HasSuffix(c.flagOut, "/") { + outFile = filepath.Join(c.flagOut, f.path) + } + if outFile == "" { + outFile = f.path + } + + updated, err := marshalYAML(f.node) + if err != nil { + return fmt.Errorf("failed to marshal yaml for %s: %w", f.path, err) + } + + final := removeNewLineChanges(string(f.contents), string(updated)) + if err := atomic.Write(f.path, outFile, strings.NewReader(final)); err != nil { + return fmt.Errorf("failed to save file %s: %w", outFile, err) + } } return nil diff --git a/command/update.go b/command/update.go index 9fdf62cbc6..36f657f2de 100644 --- a/command/update.go +++ b/command/update.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "strings" "github.com/sethvargo/ratchet/internal/atomic" @@ -15,10 +16,12 @@ import ( const updateCommandDesc = `Update all pinned versions to the latest value` const updateCommandHelp = ` +Usage: ratchet update [FILE...] + The "update" command unpins any pinned versions, resolves the unpinned version constraint to the latest available value, and then re-pins the versions. -This command will pin to the latest available version that satifies the original +This command will pin to the latest available version that satisfies the original constraint. To upgrade to versions beyond the contraint (e.g. v2 -> v3), you must manually edit the file and update the unpinned comment. @@ -49,26 +52,9 @@ func (c *UpdateCommand) Flags() *flag.FlagSet { } func (c *UpdateCommand) Run(ctx context.Context, originalArgs []string) error { - f := c.Flags() - - if err := f.Parse(originalArgs); err != nil { - return fmt.Errorf("failed to parse flags: %w", err) - } - - args := f.Args() - if got := len(args); got != 1 { - return fmt.Errorf("expected exactly one argument, got %d %q", got, args) - } - - inFile := args[0] - uneditedContent, err := parseFile(inFile) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) - } - - m, err := parseYAMLFile(inFile) + args, err := parseFlags(c.Flags(), originalArgs) if err != nil { - return fmt.Errorf("failed to parse %s: %w", inFile, err) + return fmt.Errorf("failed to parse flags: %w", err) } par, err := parser.For(ctx, c.flagParser) @@ -78,37 +64,46 @@ func (c *UpdateCommand) Run(ctx context.Context, originalArgs []string) error { res, err := resolver.NewDefaultResolver(ctx) if err != nil { - return fmt.Errorf("failed to create github resolver: %w", err) + return fmt.Errorf("failed to create resolver: %w", err) } - if err := parser.Unpin(m); err != nil { - return fmt.Errorf("failed to unpin refs: %w", err) - } + fsys := os.DirFS(".") - if err := parser.Pin(ctx, res, par, m, c.flagConcurrency); err != nil { - return fmt.Errorf("failed to pin refs: %w", err) + files, err := loadYAMLFiles(fsys, args) + if err != nil { + return err } - outFile := c.flagOut - if outFile == "" { - outFile = inFile - } - if err := writeYAMLFile(inFile, outFile, m); err != nil { - return fmt.Errorf("failed to save %s: %w", outFile, err) + if len(files) > 1 && c.flagOut != "" && !strings.HasSuffix(c.flagOut, "/") { + return fmt.Errorf("-out must be a directory when pinning multiple files") } - if !keepNewlinesEnv() { - return nil + if err := parser.Unpin(ctx, files.nodes()); err != nil { + return fmt.Errorf("failed to pin refs: %w", err) } - editedContent, err := parseFile(outFile) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", outFile, err) + if err := parser.Pin(ctx, res, par, files.nodes(), c.flagConcurrency); err != nil { + return fmt.Errorf("failed to pin refs: %w", err) } - final := removeNewLineChanges(uneditedContent, editedContent) - if err := atomic.Write(inFile, outFile, strings.NewReader(final)); err != nil { - return fmt.Errorf("failed to save file %s: %w", outFile, err) + for _, f := range files { + outFile := c.flagOut + if strings.HasSuffix(c.flagOut, "/") { + outFile = filepath.Join(c.flagOut, f.path) + } + if outFile == "" { + outFile = f.path + } + + updated, err := marshalYAML(f.node) + if err != nil { + return fmt.Errorf("failed to marshal yaml for %s: %w", f.path, err) + } + + final := removeNewLineChanges(string(f.contents), string(updated)) + if err := atomic.Write(f.path, outFile, strings.NewReader(final)); err != nil { + return fmt.Errorf("failed to save file %s: %w", outFile, err) + } } return nil diff --git a/main.go b/main.go index 98190a1eaf..e8211ee0b2 100644 --- a/main.go +++ b/main.go @@ -11,16 +11,17 @@ import ( ) func main() { - if err := realMain(); err != nil { + ctx, done := signal.NotifyContext(context.Background(), + syscall.SIGINT, syscall.SIGTERM) + defer done() + + if err := realMain(ctx); err != nil { + done() fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } } -func realMain() error { - ctx, done := signal.NotifyContext(context.Background(), - syscall.SIGINT, syscall.SIGTERM) - defer done() - +func realMain(ctx context.Context) error { return command.Run(ctx, os.Args[1:]) } diff --git a/parser/actions.go b/parser/actions.go index 26b79efdb7..17c1f4a833 100644 --- a/parser/actions.go +++ b/parser/actions.go @@ -7,26 +7,35 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - "github.com/sethvargo/ratchet/resolver" ) type Actions struct{} -// Parse pulls the GitHub Actions refs from the document. -func (a *Actions) Parse(m *yaml.Node) (*RefsList, error) { +// Parse pulls the GitHub Actions refs from the documents. +func (a *Actions) Parse(nodes []*yaml.Node) (*RefsList, error) { var refs RefsList - if m == nil { - return nil, nil + for i, node := range nodes { + if err := a.parseOne(&refs, node); err != nil { + return nil, fmt.Errorf("failed to parse node %d: %w", i, err) + } + } + + return &refs, nil +} + +func (a *Actions) parseOne(refs *RefsList, node *yaml.Node) error { + if node == nil { + return nil } - if m.Kind != yaml.DocumentNode { - return nil, fmt.Errorf("expected document node, got %v", m.Kind) + if node.Kind != yaml.DocumentNode { + return fmt.Errorf("expected document node, got %v", node.Kind) } // Top-level object map - for _, docMap := range m.Content { + for _, docMap := range node.Content { if docMap.Kind != yaml.MappingNode { continue } @@ -174,5 +183,5 @@ func (a *Actions) Parse(m *yaml.Node) (*RefsList, error) { } } - return &refs, nil + return nil } diff --git a/parser/actions_test.go b/parser/actions_test.go index 7eed815a77..7246f05ca6 100644 --- a/parser/actions_test.go +++ b/parser/actions_test.go @@ -3,6 +3,8 @@ package parser import ( "reflect" "testing" + + "github.com/braydonk/yaml" ) func TestActions_Parse(t *testing.T) { @@ -99,7 +101,7 @@ runs: m := helperStringToYAML(t, tc.in) - refs, err := new(Actions).Parse(m) + refs, err := new(Actions).Parse([]*yaml.Node{m}) if err != nil { t.Fatal(err) } diff --git a/parser/circleci.go b/parser/circleci.go index 8264569096..b0e0b91abc 100644 --- a/parser/circleci.go +++ b/parser/circleci.go @@ -6,28 +6,37 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - "github.com/sethvargo/ratchet/resolver" ) type CircleCI struct{} -// Parse pulls the CircleCI refs from the document. Unfortunately it does not +// Parse pulls the CircleCI refs from the documents. Unfortunately it does not // process "orbs" because there is no documented API for resolving orbs to an // absolute version. -func (C *CircleCI) Parse(m *yaml.Node) (*RefsList, error) { +func (c *CircleCI) Parse(nodes []*yaml.Node) (*RefsList, error) { var refs RefsList - if m == nil { - return nil, nil + for i, node := range nodes { + if err := c.parseOne(&refs, node); err != nil { + return nil, fmt.Errorf("failed to parse node %d: %w", i, err) + } + } + + return &refs, nil +} + +func (c *CircleCI) parseOne(refs *RefsList, node *yaml.Node) error { + if node == nil { + return nil } - if m.Kind != yaml.DocumentNode { - return nil, fmt.Errorf("expected document node, got %v", m.Kind) + if node.Kind != yaml.DocumentNode { + return fmt.Errorf("expected document node, got %v", node.Kind) } // Top-level object map - for _, docMap := range m.Content { + for _, docMap := range node.Content { if docMap.Kind != yaml.MappingNode { continue } @@ -74,5 +83,5 @@ func (C *CircleCI) Parse(m *yaml.Node) (*RefsList, error) { } } - return &refs, nil + return nil } diff --git a/parser/circleci_test.go b/parser/circleci_test.go index 1f62e8c8eb..3c16ab0199 100644 --- a/parser/circleci_test.go +++ b/parser/circleci_test.go @@ -3,6 +3,8 @@ package parser import ( "reflect" "testing" + + "github.com/braydonk/yaml" ) func TestCircleCI_Parse(t *testing.T) { @@ -56,7 +58,7 @@ jobs: m := helperStringToYAML(t, tc.in) - refs, err := new(CircleCI).Parse(m) + refs, err := new(CircleCI).Parse([]*yaml.Node{m}) if err != nil { t.Fatal(err) } diff --git a/parser/cloudbuild.go b/parser/cloudbuild.go index b5979f775e..4acc73367d 100644 --- a/parser/cloudbuild.go +++ b/parser/cloudbuild.go @@ -6,26 +6,35 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - "github.com/sethvargo/ratchet/resolver" ) type CloudBuild struct{} -// Parse pulls the Google Cloud Build refs from the document. -func (c *CloudBuild) Parse(m *yaml.Node) (*RefsList, error) { +// Parse pulls the Google Cloud Build refs from the documents. +func (c *CloudBuild) Parse(nodes []*yaml.Node) (*RefsList, error) { var refs RefsList - if m == nil { - return nil, nil + for i, node := range nodes { + if err := c.parseOne(&refs, node); err != nil { + return nil, fmt.Errorf("failed to parse node %d: %w", i, err) + } + } + + return &refs, nil +} + +func (c *CloudBuild) parseOne(refs *RefsList, node *yaml.Node) error { + if node == nil { + return nil } - if m.Kind != yaml.DocumentNode { - return nil, fmt.Errorf("expected document node, got %v", m.Kind) + if node.Kind != yaml.DocumentNode { + return fmt.Errorf("expected document node, got %v", node.Kind) } // Top-level object map - for _, docMap := range m.Content { + for _, docMap := range node.Content { if docMap.Kind != yaml.MappingNode { continue } @@ -59,5 +68,5 @@ func (c *CloudBuild) Parse(m *yaml.Node) (*RefsList, error) { } } - return &refs, nil + return nil } diff --git a/parser/cloudbuild_test.go b/parser/cloudbuild_test.go index 20546d5574..12d7cb62a5 100644 --- a/parser/cloudbuild_test.go +++ b/parser/cloudbuild_test.go @@ -3,6 +3,8 @@ package parser import ( "reflect" "testing" + + "github.com/braydonk/yaml" ) func TestCloudBuild_Parse(t *testing.T) { @@ -42,7 +44,7 @@ steps: m := helperStringToYAML(t, tc.in) - refs, err := new(CloudBuild).Parse(m) + refs, err := new(CloudBuild).Parse([]*yaml.Node{m}) if err != nil { t.Fatal(err) } diff --git a/parser/drone.go b/parser/drone.go index eb71e16442..a02c4bb15c 100644 --- a/parser/drone.go +++ b/parser/drone.go @@ -6,26 +6,34 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - "github.com/sethvargo/ratchet/resolver" ) type Drone struct{} -// Parse pulls the Drone Ci refs from the document. -func (D *Drone) Parse(m *yaml.Node) (*RefsList, error) { +// Parse pulls the Drone Ci refs from the documents. +func (d *Drone) Parse(nodes []*yaml.Node) (*RefsList, error) { var refs RefsList - if m == nil { - return nil, nil + for i, node := range nodes { + if err := d.parseOne(&refs, node); err != nil { + return nil, fmt.Errorf("failed to parse node %d: %w", i, err) + } } - if m.Kind != yaml.DocumentNode { - return nil, fmt.Errorf("expected document node, got %v", m.Kind) + return &refs, nil +} + +func (d *Drone) parseOne(refs *RefsList, node *yaml.Node) error { + if node == nil { + return nil } - for _, docMap := range m.Content { + if node.Kind != yaml.DocumentNode { + return fmt.Errorf("expected document node, got %v", node.Kind) + } + for _, docMap := range node.Content { if docMap.Kind != yaml.MappingNode { continue } @@ -58,5 +66,5 @@ func (D *Drone) Parse(m *yaml.Node) (*RefsList, error) { } } - return &refs, nil + return nil } diff --git a/parser/drone_test.go b/parser/drone_test.go index de6d089bf2..e7c1b7e09c 100644 --- a/parser/drone_test.go +++ b/parser/drone_test.go @@ -3,6 +3,8 @@ package parser import ( "reflect" "testing" + + "github.com/braydonk/yaml" ) func TestDrone_Parse(t *testing.T) { @@ -26,7 +28,7 @@ jobs: steps: - name: git image: alpine/git - + - name: test image: mysql `, @@ -45,7 +47,7 @@ steps: m := helperStringToYAML(t, tc.in) - refs, err := new(Drone).Parse(m) + refs, err := new(Drone).Parse([]*yaml.Node{m}) if err != nil { t.Fatal(err) } diff --git a/parser/gitlabci.go b/parser/gitlabci.go index 8a52b37a08..118754ca15 100644 --- a/parser/gitlabci.go +++ b/parser/gitlabci.go @@ -6,7 +6,6 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - "github.com/sethvargo/ratchet/resolver" ) @@ -14,8 +13,19 @@ type GitLabCI struct{} // Parse pulls the image references from GitLab CI configuration files. It does // not support references with variables. -func (C *GitLabCI) Parse(m *yaml.Node) (*RefsList, error) { +func (c *GitLabCI) Parse(nodes []*yaml.Node) (*RefsList, error) { var refs RefsList + + for i, node := range nodes { + if err := c.parseOne(&refs, node); err != nil { + return nil, fmt.Errorf("failed to parse node %d: %w", i, err) + } + } + + return &refs, nil +} + +func (c *GitLabCI) parseOne(refs *RefsList, m *yaml.Node) error { var imageRef *yaml.Node // GitLab CI global top level keywords @@ -28,11 +38,11 @@ func (C *GitLabCI) Parse(m *yaml.Node) (*RefsList, error) { } if m == nil { - return nil, nil + return nil } if m.Kind != yaml.DocumentNode { - return nil, fmt.Errorf("expected document node, got %v", m.Kind) + return fmt.Errorf("expected document node, got %v", m.Kind) } // Top-level object map @@ -42,7 +52,6 @@ func (C *GitLabCI) Parse(m *yaml.Node) (*RefsList, error) { } // jobs names for i, keysMap := range docMap.Content { - // exclude global keywords if _, hit := globalKeywords[keysMap.Value]; hit || (keysMap.Value == "") { continue @@ -55,7 +64,6 @@ func (C *GitLabCI) Parse(m *yaml.Node) (*RefsList, error) { for k, property := range job.Content { if property.Value == "image" { - image := job.Content[k+1] // match image reference with name key @@ -77,5 +85,5 @@ func (C *GitLabCI) Parse(m *yaml.Node) (*RefsList, error) { } } - return &refs, nil + return nil } diff --git a/parser/gitlabci_test.go b/parser/gitlabci_test.go index 487d5ad934..8c3af0750f 100644 --- a/parser/gitlabci_test.go +++ b/parser/gitlabci_test.go @@ -4,6 +4,8 @@ import ( "fmt" "reflect" "testing" + + "github.com/braydonk/yaml" ) func TestGitLabCI_Parse(t *testing.T) { @@ -104,7 +106,7 @@ job2: m := helperStringToYAML(t, tc.in) - refs, err := new(GitLabCI).Parse(m) + refs, err := new(GitLabCI).Parse([]*yaml.Node{m}) if err != nil { fmt.Println(refs) t.Fatal(err) diff --git a/parser/parser.go b/parser/parser.go index 0f1e553c72..b1a85f89eb 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -11,10 +11,8 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - - "golang.org/x/sync/semaphore" - "github.com/sethvargo/ratchet/resolver" + "golang.org/x/sync/semaphore" ) const ( @@ -25,7 +23,7 @@ const ( // Parser defines an interface which parses references out of the given yaml // node. type Parser interface { - Parse(m *yaml.Node) (*RefsList, error) + Parse(nodes []*yaml.Node) (*RefsList, error) } var parserFactory = map[string]func() Parser{ @@ -58,8 +56,8 @@ func List() []string { // Check iterates over all references in the yaml and checks if they are pinned // to an absolute reference. It ignores "ratchet:exclude" nodes from the lookup. -func Check(ctx context.Context, parser Parser, m *yaml.Node) error { - refsList, err := parser.Parse(m) +func Check(ctx context.Context, parser Parser, nodes []*yaml.Node) error { + refsList, err := parser.Parse(nodes) if err != nil { return err } @@ -67,6 +65,12 @@ func Check(ctx context.Context, parser Parser, m *yaml.Node) error { var unpinned []string for ref, nodes := range refs { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + ref = resolver.DenormalizeRef(ref) // Pre-filter any nodes that should be excluded from the lookup. @@ -95,8 +99,8 @@ func Check(ctx context.Context, parser Parser, m *yaml.Node) error { // Pin extracts all references from the given YAML document and resolves them // using the given resolver, updating the associated YAML nodes. -func Pin(ctx context.Context, res resolver.Resolver, parser Parser, m *yaml.Node, concurrency int64) error { - refsList, err := parser.Parse(m) +func Pin(ctx context.Context, res resolver.Resolver, parser Parser, nodes []*yaml.Node, concurrency int64) error { + refsList, err := parser.Parse(nodes) if err != nil { return err } @@ -175,20 +179,22 @@ func Pin(ctx context.Context, res resolver.Resolver, parser Parser, m *yaml.Node // // This function does not make any outbound network calls and relies solely on // information in the document. -func Unpin(m *yaml.Node) error { - if m == nil { - return nil - } +func Unpin(ctx context.Context, nodes []*yaml.Node) error { + for _, node := range nodes { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } - if m.LineComment != "" && !shouldExclude(m.LineComment) { - if v, rest := extractOriginalFromComment(m.LineComment); v != "" { - m.Value = v - m.LineComment = rest + if node.LineComment != "" && !shouldExclude(node.LineComment) { + if v, rest := extractOriginalFromComment(node.LineComment); v != "" { + node.Value = v + node.LineComment = rest + } } - } - for _, child := range m.Content { - if err := Unpin(child); err != nil { + if err := Unpin(ctx, node.Content); err != nil { return err } } diff --git a/parser/parser_test.go b/parser/parser_test.go index 5d4bf3ef1f..0708dce18a 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -9,7 +9,6 @@ import ( // Using banydonk/yaml instead of the default yaml pkg because the default // pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737 "github.com/braydonk/yaml" - "github.com/sethvargo/ratchet/resolver" ) @@ -69,7 +68,7 @@ jobs: m := helperStringToYAML(t, tc.in) - if err := Check(ctx, par, m); err != nil { + if err := Check(ctx, par, []*yaml.Node{m}); err != nil { if tc.err == "" { t.Fatal(err) } else { @@ -211,7 +210,7 @@ jobs: m := helperStringToYAML(t, tc.in) - if err := Pin(ctx, res, par, m, 2); err != nil { + if err := Pin(ctx, res, par, []*yaml.Node{m}, 2); err != nil { if tc.err == "" { t.Fatal(err) } else { @@ -235,6 +234,8 @@ jobs: func TestUnpin(t *testing.T) { t.Parallel() + ctx := context.Background() + cases := []struct { name string in string @@ -283,7 +284,7 @@ func TestUnpin(t *testing.T) { m := helperStringToYAML(t, tc.in) - if err := Unpin(m); err != nil { + if err := Unpin(ctx, []*yaml.Node{m}); err != nil { t.Fatal(err) }