Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for multiple output files in different formats #732

Merged
merged 24 commits into from
Jan 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ may attempt to expand wildcards, so put those parameters in single quotes, like:

### Output formats

The output format for Syft is configurable as well:
The output format for Syft is configurable as well using the
`-o` (or `--output`) option:

```
syft packages <image> -o <format>
Expand All @@ -127,6 +128,15 @@ Where the `formats` available are:
- `spdx-json`: A JSON report conforming to the [SPDX 2.2 JSON Schema](https://github.com/spdx/spdx-spec/blob/v2.2/schemas/spdx-schema.json).
- `table`: A columnar summary (default).

#### Multiple outputs

Syft can also output _multiple_ files in differing formats by appending
`=<file>` to the option, for example to output Syft JSON and SPDX JSON:

```shell
syft packages <image> -o json=sbom.syft.json -o spdx-json=sbom.spdx.json
```

## Private Registry Authentication

### Local Docker Credentials
Expand Down Expand Up @@ -221,8 +231,12 @@ Configuration search paths:
Configuration options (example values are the default):

```yaml
# the output format of the SBOM report (options: table, text, json)
# same as -o ; SYFT_OUTPUT env var
# the output format(s) of the SBOM report (options: table, text, json, spdx, ...)
# same as -o, --output, and SYFT_OUTPUT env var
# to specify multiple output files in differing formats, use a list:
# output:
# - "json=<syft-json-output-file>"
# - "spdx-json=<spdx-json-output-file>"
output: "table"

# suppress all output (except for the SBOM report)
Expand All @@ -238,8 +252,8 @@ check-for-app-update: true

# a list of globs to exclude from scanning. same as --exclude ; for example:
# exclude:
# - '/etc/**'
# - './out/**/*.json'
# - "/etc/**"
kzantow marked this conversation as resolved.
Show resolved Hide resolved
# - "./out/**/*.json"
exclude:

# cataloging packages is exposed through the packages and power-user subcommands
Expand Down
10 changes: 5 additions & 5 deletions cmd/event_loop_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func Test_eventLoop_gracefulExit(t *testing.T) {
t.Cleanup(testBus.Close)

finalEvent := partybus.Event{
Type: event.PresenterReady,
Type: event.Exit,
}

worker := func() <-chan error {
Expand Down Expand Up @@ -182,7 +182,7 @@ func Test_eventLoop_unsubscribeError(t *testing.T) {
t.Cleanup(testBus.Close)

finalEvent := partybus.Event{
Type: event.PresenterReady,
Type: event.Exit,
}

worker := func() <-chan error {
Expand Down Expand Up @@ -251,8 +251,8 @@ func Test_eventLoop_handlerError(t *testing.T) {
t.Cleanup(testBus.Close)

finalEvent := partybus.Event{
Type: event.PresenterReady,
Error: fmt.Errorf("unable to create presenter"),
Type: event.Exit,
Error: fmt.Errorf("an exit error occured"),
}

worker := func() <-chan error {
Expand Down Expand Up @@ -376,7 +376,7 @@ func Test_eventLoop_uiTeardownError(t *testing.T) {
t.Cleanup(testBus.Close)

finalEvent := partybus.Event{
Type: event.PresenterReady,
Type: event.Exit,
}

worker := func() <-chan error {
Expand Down
72 changes: 72 additions & 0 deletions cmd/output_writer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package cmd

import (
"fmt"
"strings"

"github.com/anchore/syft/internal/formats"
"github.com/anchore/syft/internal/output"
"github.com/anchore/syft/syft/format"
"github.com/anchore/syft/syft/sbom"
"github.com/hashicorp/go-multierror"
)

// makeWriter creates a sbom.Writer for output or returns an error. this will either return a valid writer
// or an error but neither both and if there is no error, sbom.Writer.Close() should be called
func makeWriter(outputs []string, defaultFile string) (sbom.Writer, error) {
outputOptions, err := parseOptions(outputs, defaultFile)
if err != nil {
return nil, err
}

writer, err := output.MakeWriter(outputOptions...)
if err != nil {
return nil, err
}

return writer, nil
}

// parseOptions utility to parse command-line option strings and retain the existing behavior of default format and file
func parseOptions(outputs []string, defaultFile string) (out []output.WriterOption, errs error) {
// always should have one option -- we generally get the default of "table", but just make sure
if len(outputs) == 0 {
outputs = append(outputs, string(format.TableOption))
}

for _, name := range outputs {
name = strings.TrimSpace(name)

// split to at most two parts for <format>=<file>
parts := strings.SplitN(name, "=", 2)

// the format option is the first part
name = parts[0]

// default to the --file or empty string if not specified
file := defaultFile

// If a file is specified as part of the output option, use that
if len(parts) > 1 {
file = parts[1]
}

option := format.ParseOption(name)
if option == format.UnknownFormatOption {
errs = multierror.Append(errs, fmt.Errorf("bad output format: '%s'", name))
continue
}

encoder := formats.ByOption(option)
if encoder == nil {
errs = multierror.Append(errs, fmt.Errorf("unknown format: %s", outputFormat))
continue
}

out = append(out, output.WriterOption{
Format: *encoder,
Path: file,
})
}
return out, errs
}
78 changes: 78 additions & 0 deletions cmd/output_writer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package cmd

import (
"fmt"
"strings"
"testing"

"github.com/stretchr/testify/assert"
)

func TestOutputWriterConfig(t *testing.T) {
tmp := t.TempDir() + "/"

tests := []struct {
outputs []string
file string
err bool
expected []string
}{
{
outputs: []string{},
expected: []string{""},
},
{
outputs: []string{"json"},
expected: []string{""},
},
{
file: "test-1.json",
expected: []string{"test-1.json"},
},
{
outputs: []string{"json=test-2.json"},
expected: []string{"test-2.json"},
},
{
outputs: []string{"json=test-3-1.json", "spdx-json=test-3-2.json"},
expected: []string{"test-3-1.json", "test-3-2.json"},
},
{
outputs: []string{"text", "json=test-4.json"},
expected: []string{"", "test-4.json"},
},
}

for _, test := range tests {
t.Run(fmt.Sprintf("%s/%s", test.outputs, test.file), func(t *testing.T) {
outputs := test.outputs
for i, val := range outputs {
outputs[i] = strings.Replace(val, "=", "="+tmp, 1)
}

file := test.file
if file != "" {
file = tmp + file
}

_, err := makeWriter(test.outputs, file)

if test.err {
assert.Error(t, err)
return
} else {
assert.NoError(t, err)
}

for _, expected := range test.expected {
kzantow marked this conversation as resolved.
Show resolved Hide resolved
if expected != "" {
assert.FileExists(t, tmp+expected)
} else if file != "" {
assert.FileExists(t, file)
} else {
assert.NoFileExists(t, expected)
}
}
})
}
}
53 changes: 19 additions & 34 deletions cmd/packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/anchore"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/formats"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/ui"
"github.com/anchore/syft/internal/version"
Expand Down Expand Up @@ -51,8 +50,7 @@ const (
)

var (
packagesPresenterOpt format.Option
packagesCmd = &cobra.Command{
packagesCmd = &cobra.Command{
Use: "packages [SOURCE]",
Short: "Generate a package SBOM",
Long: "Generate a packaged-based Software Bill Of Materials (SBOM) from container images and filesystems",
Expand All @@ -63,14 +61,7 @@ var (
Args: validateInputArgs,
SilenceUsage: true,
SilenceErrors: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
// set the presenter
presenterOption := format.ParseOption(appConfig.Output)
if presenterOption == format.UnknownFormatOption {
return fmt.Errorf("bad --output value '%s'", appConfig.Output)
}
packagesPresenterOpt = presenterOption

PreRunE: func(cmd *cobra.Command, args []string) (err error) {
if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem {
return fmt.Errorf("cannot profile CPU and memory simultaneously")
}
Expand Down Expand Up @@ -102,14 +93,14 @@ func setPackageFlags(flags *pflag.FlagSet) {
"scope", "s", cataloger.DefaultSearchConfig().Scope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))

flags.StringP(
"output", "o", string(format.TableOption),
fmt.Sprintf("report output formatter, options=%v", format.AllOptions),
flags.StringArrayP(
"output", "o", []string{string(format.TableOption)},
fmt.Sprintf("report output format, options=%v", format.AllOptions),
)

flags.StringP(
"file", "", "",
"file to write the report output to (default is STDOUT)",
"file to write the default report output to (default is STDOUT)",
Copy link
Contributor Author

@kzantow kzantow Jan 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be deprecated? if so, how would that be done? just updates to this and the readme?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is -o intended to replace --file?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really know, but the functionality is redundant. it would seem outputting a table to a file is fairly useless, so generally a user would have to do -o spdx-json --file <output-file>, for example. But this is already covered by the -o spdx-json=<output-file>...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see your point on redundancy.

My take: I think it makes sense to leave the --file flag as it's not harming anything by leaving it. We probably don't need to add a deprecation notice if we aren't removing it (we could always decide to remove it in the future).

)

// Upload options //////////////////////////////////////////////////////////
Expand Down Expand Up @@ -210,26 +201,26 @@ func validateInputArgs(cmd *cobra.Command, args []string) error {
}

func packagesExec(_ *cobra.Command, args []string) error {
// could be an image or a directory, with or without a scheme
userInput := args[0]
writer, err := makeWriter(appConfig.Output, appConfig.File)
if err != nil {
return err
}

reporter, closer, err := reportWriter()
defer func() {
if err := closer(); err != nil {
log.Warnf("unable to write to report destination: %+v", err)
if err := writer.Close(); err != nil {
log.Warnf("unable to write to report destination: %w", err)
}
}()

if err != nil {
return err
}
// could be an image or a directory, with or without a scheme
userInput := args[0]

return eventLoop(
packagesExecWorker(userInput),
packagesExecWorker(userInput, writer),
setupSignals(),
eventSubscription,
stereoscope.Cleanup,
ui.Select(isVerbose(), appConfig.Quiet, reporter)...,
ui.Select(isVerbose(), appConfig.Quiet)...,
)
}

Expand All @@ -244,7 +235,7 @@ func isVerbose() (result bool) {
return appConfig.CliOptions.Verbosity > 0 || isPipedInput
}

func packagesExecWorker(userInput string) <-chan error {
func packagesExecWorker(userInput string, writer sbom.Writer) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
Expand All @@ -255,12 +246,6 @@ func packagesExecWorker(userInput string) <-chan error {
return
}

f := formats.ByOption(packagesPresenterOpt)
if f == nil {
errs <- fmt.Errorf("unknown format: %s", packagesPresenterOpt)
return
}

src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
if err != nil {
errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
Expand Down Expand Up @@ -296,8 +281,8 @@ func packagesExecWorker(userInput string) <-chan error {
}

bus.Publish(partybus.Event{
Type: event.PresenterReady,
Value: f.Presenter(s),
Type: event.Exit,
Value: func() error { return writer.Write(s) },
})
}()
return errs
Expand Down
Loading