From 64bb191225c27c2dcfe1c6819dba0250a1632faa Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 12:56:13 +0600 Subject: [PATCH 01/16] fear(parser): add parser for environment.yaml --- .../parser/conda/environment/parse.go | 64 +++++++++++++++++++ .../parser/conda/environment/parse_test.go | 59 +++++++++++++++++ .../conda/environment/testdata/happy.yaml | 34 ++++++++++ 3 files changed, 157 insertions(+) create mode 100644 pkg/dependency/parser/conda/environment/parse.go create mode 100644 pkg/dependency/parser/conda/environment/parse_test.go create mode 100644 pkg/dependency/parser/conda/environment/testdata/happy.yaml diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go new file mode 100644 index 000000000000..ce4f07d94473 --- /dev/null +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -0,0 +1,64 @@ +package environment + +import ( + "github.com/aquasecurity/trivy/pkg/dependency/types" + "github.com/aquasecurity/trivy/pkg/log" + xio "github.com/aquasecurity/trivy/pkg/x/io" + "golang.org/x/xerrors" + "gopkg.in/yaml.v3" + "sort" + "strings" +) + +type environment struct { + Dependencies []Dependency `yaml:"dependencies"` +} + +type Dependency struct { + Value string + Line int +} + +type Parser struct { + logger *log.Logger +} + +func NewParser() types.Parser { + return &Parser{ + logger: log.WithPrefix("conda"), + } +} + +// TODO add comment +func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) { + var env environment + if err := yaml.NewDecoder(r).Decode(&env); err != nil { + return nil, nil, xerrors.Errorf("unable to decode conda environment.yml file: %w", err) + } + + var libs []types.Library + for _, dep := range env.Dependencies { + lib, err := p.toLibrary(dep) + if err != nil { + return nil, nil, xerrors.Errorf("unable to parse dependency: %w", err) + } + libs = append(libs, lib) + } + + sort.Sort(types.Libraries(libs)) + return libs, nil, nil +} + +func (p *Parser) toLibrary(dep Dependency) (types.Library, error) { + ss := strings.Split(dep.Value, "=") + if len(ss) == 1 { + p.logger.Debug("Unable to detect version", log.String("dependency", dep.Value)) + } + return types.Library{}, nil +} + +func (d *Dependency) UnmarshalYAML(node *yaml.Node) error { + d.Value = node.Value + d.Line = node.Line + return nil +} diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go new file mode 100644 index 000000000000..dbc45ef0468f --- /dev/null +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -0,0 +1,59 @@ +package environment_test + +import ( + "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aquasecurity/trivy/pkg/dependency/types" +) + +func TestParse(t *testing.T) { + tests := []struct { + name string + input string + want []types.Library + wantErr string + }{ + { + name: "happy path", + input: "testdata/happy.yaml", + want: []types.Library{ + { + Name: "_libgcc_mutex", + Version: "0.1", + }, + }, + }, + { + name: "invalid_json", + input: "testdata/invalid_json.json", + wantErr: "JSON decode error: invalid character", + }, + { + name: "invalid_package", + input: "testdata/invalid_package.json", + wantErr: "unable to parse conda package", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.input) + require.NoError(t, err) + defer f.Close() + + got, _, err := environment.NewParser().Parse(f) + + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/dependency/parser/conda/environment/testdata/happy.yaml b/pkg/dependency/parser/conda/environment/testdata/happy.yaml new file mode 100644 index 000000000000..f624326d9eae --- /dev/null +++ b/pkg/dependency/parser/conda/environment/testdata/happy.yaml @@ -0,0 +1,34 @@ +name: test-env +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=51_gnu + - blas=1.0=openblas + - bzip2=1.0.8=h998d150_5 + - ca-certificates=2024.3.11=hd43f75c_0 + - expat=2.6.2=h419075a_0 + - ld_impl_linux-aarch64=2.38=h8131f2d_1 + - libffi=3.4.4=h419075a_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=11.2.0=h6e398d7_1 + - libgfortran5=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libopenblas=0.3.21=hc2e42e2_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libuuid=1.41.5=h998d150_0 + - ncurses=6.4=h419075a_0 + - numpy=1.26.4=py312hdb1dca2_0 + - numpy-base=1.26.4=py312h6f96b8b_0 + - openssl=3.0.13=h2f4d8fa_0 + - pip=23.3.1=py312hd43f75c_0 + - python=3.12.3=h8edadfe_0 + - readline=8.2=h998d150_0 + - scipy=1.12.0=py312hdb1dca2_0 + - setuptools=68.2.2=py312hd43f75c_0 + - sqlite=3.41.2=h998d150_0 + - tk=8.6.12=h241ca14_0 + - tzdata=2024a=h04d1e81_0 + - wheel=0.41.2=py312hd43f75c_0 + - xz=5.4.6=h998d150_0 + - zlib=1.2.13=h998d150_0 \ No newline at end of file From 0894f34ddf6b807dc88c26a2d955007420a65736 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 13:53:56 +0600 Subject: [PATCH 02/16] refactor(parser): update `toLibrary` function --- .../parser/conda/environment/parse.go | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go index ce4f07d94473..c8748019de7b 100644 --- a/pkg/dependency/parser/conda/environment/parse.go +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -29,7 +29,6 @@ func NewParser() types.Parser { } } -// TODO add comment func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) { var env environment if err := yaml.NewDecoder(r).Decode(&env); err != nil { @@ -50,11 +49,29 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, } func (p *Parser) toLibrary(dep Dependency) (types.Library, error) { + // Default format for files created using the `conda Export` command: `== + // e.g. `bzip2=1.0.8=h998d150_5` + // But it is also possible to set only the dependency name ss := strings.Split(dep.Value, "=") + + lib := types.Library{ + Name: ss[0], + Locations: types.Locations{ + { + StartLine: dep.Line, + EndLine: dep.Line, + }, + }, + } + + // Version can be omitted if len(ss) == 1 { - p.logger.Debug("Unable to detect version", log.String("dependency", dep.Value)) + p.logger.Warn("Unable to detect the version as it is not pinned", log.String("name", dep.Value)) + return lib, nil } - return types.Library{}, nil + + lib.Version = ss[1] + return lib, nil } func (d *Dependency) UnmarshalYAML(node *yaml.Node) error { From cc797be8d087bab10a7292e9ff367b78d6eeadce Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 13:54:10 +0600 Subject: [PATCH 03/16] test(parser): update/add tests --- .../parser/conda/environment/parse_test.go | 47 +++++++++++++++---- .../conda/environment/testdata/happy.yaml | 31 ++---------- .../conda/environment/testdata/invalid.yaml | 1 + 3 files changed, 42 insertions(+), 37 deletions(-) create mode 100644 pkg/dependency/parser/conda/environment/testdata/invalid.yaml diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go index dbc45ef0468f..6b133289d732 100644 --- a/pkg/dependency/parser/conda/environment/parse_test.go +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -23,20 +23,49 @@ func TestParse(t *testing.T) { input: "testdata/happy.yaml", want: []types.Library{ { - Name: "_libgcc_mutex", - Version: "0.1", + Name: "_libgcc_mutex", + Locations: types.Locations{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "_openmp_mutex", + Version: "5.1", + Locations: types.Locations{ + { + StartLine: 6, + EndLine: 6, + }, + }, + }, { + Name: "blas", + Version: "1.0", + Locations: types.Locations{ + { + StartLine: 7, + EndLine: 7, + }, + }, + }, + { + Name: "bzip2", + Version: "1.0.8", + Locations: types.Locations{ + { + StartLine: 8, + EndLine: 8, + }, + }, }, }, }, { name: "invalid_json", - input: "testdata/invalid_json.json", - wantErr: "JSON decode error: invalid character", - }, - { - name: "invalid_package", - input: "testdata/invalid_package.json", - wantErr: "unable to parse conda package", + input: "testdata/invalid.yaml", + wantErr: "unable to decode conda environment.yml file", }, } for _, tt := range tests { diff --git a/pkg/dependency/parser/conda/environment/testdata/happy.yaml b/pkg/dependency/parser/conda/environment/testdata/happy.yaml index f624326d9eae..62cd7ff599bd 100644 --- a/pkg/dependency/parser/conda/environment/testdata/happy.yaml +++ b/pkg/dependency/parser/conda/environment/testdata/happy.yaml @@ -2,33 +2,8 @@ name: test-env channels: - defaults dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=51_gnu + - _libgcc_mutex + - _openmp_mutex=5.1 - blas=1.0=openblas - bzip2=1.0.8=h998d150_5 - - ca-certificates=2024.3.11=hd43f75c_0 - - expat=2.6.2=h419075a_0 - - ld_impl_linux-aarch64=2.38=h8131f2d_1 - - libffi=3.4.4=h419075a_0 - - libgcc-ng=11.2.0=h1234567_1 - - libgfortran-ng=11.2.0=h6e398d7_1 - - libgfortran5=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libopenblas=0.3.21=hc2e42e2_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libuuid=1.41.5=h998d150_0 - - ncurses=6.4=h419075a_0 - - numpy=1.26.4=py312hdb1dca2_0 - - numpy-base=1.26.4=py312h6f96b8b_0 - - openssl=3.0.13=h2f4d8fa_0 - - pip=23.3.1=py312hd43f75c_0 - - python=3.12.3=h8edadfe_0 - - readline=8.2=h998d150_0 - - scipy=1.12.0=py312hdb1dca2_0 - - setuptools=68.2.2=py312hd43f75c_0 - - sqlite=3.41.2=h998d150_0 - - tk=8.6.12=h241ca14_0 - - tzdata=2024a=h04d1e81_0 - - wheel=0.41.2=py312hd43f75c_0 - - xz=5.4.6=h998d150_0 - - zlib=1.2.13=h998d150_0 \ No newline at end of file +prefix: /opt/conda/envs/test-env diff --git a/pkg/dependency/parser/conda/environment/testdata/invalid.yaml b/pkg/dependency/parser/conda/environment/testdata/invalid.yaml new file mode 100644 index 000000000000..e466dcbd8e8f --- /dev/null +++ b/pkg/dependency/parser/conda/environment/testdata/invalid.yaml @@ -0,0 +1 @@ +invalid \ No newline at end of file From 26d3908820eb515acdcc8b36db4963860134f62f Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 14:43:22 +0600 Subject: [PATCH 04/16] feat(fanal): add environmentAnalyzer --- .../parser/conda/environment/parse_test.go | 2 +- pkg/fanal/analyzer/all/import.go | 1 + pkg/fanal/analyzer/const.go | 1 + .../language/conda/environment/environment.go | 39 ++++++ .../conda/environment/environment_test.go | 131 ++++++++++++++++++ .../environment/testdata/environment.yaml | 9 ++ .../conda/environment/testdata/invalid.yaml | 1 + pkg/fanal/types/const.go | 1 + 8 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 pkg/fanal/analyzer/language/conda/environment/environment.go create mode 100644 pkg/fanal/analyzer/language/conda/environment/environment_test.go create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/environment.yaml create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/invalid.yaml diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go index 6b133289d732..a1cefc43c3da 100644 --- a/pkg/dependency/parser/conda/environment/parse_test.go +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -1,13 +1,13 @@ package environment_test import ( - "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment" "github.com/aquasecurity/trivy/pkg/dependency/types" ) diff --git a/pkg/fanal/analyzer/all/import.go b/pkg/fanal/analyzer/all/import.go index 443ebc6bdfb8..acd0b4cc70e2 100644 --- a/pkg/fanal/analyzer/all/import.go +++ b/pkg/fanal/analyzer/all/import.go @@ -8,6 +8,7 @@ import ( _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/imgconf/dockerfile" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/imgconf/secret" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/c/conan" + _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/conda/environment" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/conda/meta" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dart/pub" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dotnet/deps" diff --git a/pkg/fanal/analyzer/const.go b/pkg/fanal/analyzer/const.go index 29ed8027118f..3dc37052aaae 100644 --- a/pkg/fanal/analyzer/const.go +++ b/pkg/fanal/analyzer/const.go @@ -69,6 +69,7 @@ const ( // Conda TypeCondaPkg Type = "conda-pkg" + TypeCondaEnv Type = "conda-environment" // Python TypePythonPkg Type = "python-pkg" diff --git a/pkg/fanal/analyzer/language/conda/environment/environment.go b/pkg/fanal/analyzer/language/conda/environment/environment.go new file mode 100644 index 000000000000..3b700da96300 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/environment.go @@ -0,0 +1,39 @@ +package environment + +import ( + "context" + "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" + "github.com/aquasecurity/trivy/pkg/fanal/types" + "golang.org/x/xerrors" + "os" + "path/filepath" +) + +func init() { + analyzer.RegisterAnalyzer(&environmentAnalyzer{}) +} + +const version = 1 + +type environmentAnalyzer struct{} + +func (a environmentAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { + res, err := language.Analyze(types.CondaEnv, input.FilePath, input.Content, environment.NewParser()) + if err != nil { + return nil, xerrors.Errorf("unable to parse environment.yaml: %w", err) + } + return res, nil +} +func (a environmentAnalyzer) Required(filePath string, _ os.FileInfo) bool { + return filepath.Base(filePath) == "environment.yaml" || filepath.Base(filePath) == "environment.yml" +} + +func (a environmentAnalyzer) Type() analyzer.Type { + return analyzer.TypeCondaEnv +} + +func (a environmentAnalyzer) Version() int { + return version +} diff --git a/pkg/fanal/analyzer/language/conda/environment/environment_test.go b/pkg/fanal/analyzer/language/conda/environment/environment_test.go new file mode 100644 index 000000000000..02139200bd15 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/environment_test.go @@ -0,0 +1,131 @@ +package environment + +import ( + "context" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "os" + "testing" +) + +func Test_environmentAnalyzer_Analyze(t *testing.T) { + tests := []struct { + name string + inputFile string + want *analyzer.AnalysisResult + wantErr string + }{ + { + name: "happy path", + inputFile: "testdata/environment.yaml", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.CondaEnv, + FilePath: "testdata/environment.yaml", + Libraries: types.Packages{ + { + Name: "_libgcc_mutex", + Locations: []types.Location{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "_openmp_mutex", + Version: "5.1", + Locations: []types.Location{ + { + StartLine: 6, + EndLine: 6, + }, + }, + }, { + Name: "blas", + Version: "1.0", + Locations: []types.Location{ + { + StartLine: 7, + EndLine: 7, + }, + }, + }, + { + Name: "bzip2", + Version: "1.0.8", + Locations: []types.Location{ + { + StartLine: 8, + EndLine: 8, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "invalid", + inputFile: "testdata/invalid.yaml", + wantErr: "unable to parse environment.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.inputFile) + require.NoError(t, err) + defer f.Close() + + a := environmentAnalyzer{} + ctx := context.Background() + got, err := a.Analyze(ctx, analyzer.AnalysisInput{ + FilePath: tt.inputFile, + Content: f, + }) + + if tt.wantErr != "" { + require.Error(t, err) + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func Test_environmentAnalyzer_Required(t *testing.T) { + tests := []struct { + name string + filePath string + want bool + }{ + { + name: "happy path `yaml`", + filePath: "foo/environment.yaml", + want: true, + }, + { + name: "happy path `yml`", + filePath: "bar/environment.yaml", + want: true, + }, + { + name: "sad path `json` ", + filePath: "environment.json", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := environmentAnalyzer{} + got := a.Required(tt.filePath, nil) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/environment.yaml b/pkg/fanal/analyzer/language/conda/environment/testdata/environment.yaml new file mode 100644 index 000000000000..62cd7ff599bd --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/environment.yaml @@ -0,0 +1,9 @@ +name: test-env +channels: + - defaults +dependencies: + - _libgcc_mutex + - _openmp_mutex=5.1 + - blas=1.0=openblas + - bzip2=1.0.8=h998d150_5 +prefix: /opt/conda/envs/test-env diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/invalid.yaml b/pkg/fanal/analyzer/language/conda/environment/testdata/invalid.yaml new file mode 100644 index 000000000000..9977a2836c1a --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/invalid.yaml @@ -0,0 +1 @@ +invalid diff --git a/pkg/fanal/types/const.go b/pkg/fanal/types/const.go index b46b36a8d425..52769674539b 100644 --- a/pkg/fanal/types/const.go +++ b/pkg/fanal/types/const.go @@ -55,6 +55,7 @@ const ( Pipenv LangType = "pipenv" Poetry LangType = "poetry" CondaPkg LangType = "conda-pkg" + CondaEnv LangType = "conda-environment" PythonPkg LangType = "python-pkg" NodePkg LangType = "node-pkg" Yarn LangType = "yarn" From 8f73c4b201e066e104dea0e1e68ab46c462a6a16 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 14:44:17 +0600 Subject: [PATCH 05/16] feat(detector): add CondaEnv to drivers --- pkg/detector/library/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detector/library/driver.go b/pkg/detector/library/driver.go index e94f2b4db89f..64bd140ed57e 100644 --- a/pkg/detector/library/driver.go +++ b/pkg/detector/library/driver.go @@ -72,7 +72,7 @@ func NewDriver(libType ftypes.LangType) (Driver, bool) { // https://guides.cocoapods.org/making/making-a-cocoapod.html#cocoapods-versioning-specifics ecosystem = vulnerability.Cocoapods comparer = rubygems.Comparer{} - case ftypes.CondaPkg: + case ftypes.CondaPkg, ftypes.CondaEnv: log.Warn("Conda package is supported for SBOM, not for vulnerability scanning") return Driver{}, false case ftypes.Bitnami: From d1acd53cc5c22d0986e940a0a3dd66dafb9a5d5e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 14:53:17 +0600 Subject: [PATCH 06/16] feat(purl): add CondaEnv --- pkg/purl/purl.go | 2 +- pkg/purl/purl_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/purl/purl.go b/pkg/purl/purl.go index 59a4b99df30c..608c7b0b4029 100644 --- a/pkg/purl/purl.go +++ b/pkg/purl/purl.go @@ -432,7 +432,7 @@ func purlType(t ftypes.TargetType) string { return packageurl.TypeGem case ftypes.NuGet, ftypes.DotNetCore, ftypes.PackagesProps: return packageurl.TypeNuget - case ftypes.CondaPkg: + case ftypes.CondaPkg, ftypes.CondaEnv: return packageurl.TypeConda case ftypes.PythonPkg, ftypes.Pip, ftypes.Pipenv, ftypes.Poetry: return packageurl.TypePyPi diff --git a/pkg/purl/purl_test.go b/pkg/purl/purl_test.go index d22e010d8b22..646930ee5b76 100644 --- a/pkg/purl/purl_test.go +++ b/pkg/purl/purl_test.go @@ -131,6 +131,19 @@ func TestNewPackageURL(t *testing.T) { Version: "0.4.1", }, }, + { + name: "conda environment.yaml", + typ: ftypes.CondaEnv, + pkg: ftypes.Package{ + Name: "blas", + Version: "1.0", + }, + want: &purl.PackageURL{ + Type: packageurl.TypeConda, + Name: "blas", + Version: "1.0", + }, + }, { name: "composer package", typ: ftypes.Composer, From a263edb0794fd4fd14f24d51dec51ff7709e7c9e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 15:01:48 +0600 Subject: [PATCH 07/16] refactor(fanal): mark CondaEnv al lock file --- pkg/fanal/analyzer/const.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/fanal/analyzer/const.go b/pkg/fanal/analyzer/const.go index 3dc37052aaae..ef20482a782c 100644 --- a/pkg/fanal/analyzer/const.go +++ b/pkg/fanal/analyzer/const.go @@ -178,6 +178,7 @@ var ( TypeDotNetCore, TypePackagesProps, TypeCondaPkg, + TypeCondaEnv, TypePythonPkg, TypePip, TypePipenv, @@ -209,6 +210,7 @@ var ( TypeSwift, TypePubSpecLock, TypeMixLock, + TypeCondaEnv, } // TypeIndividualPkgs has all analyzers for individual packages From 5415d71bd04060da41aa04d70151f1a4fde35a92 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 15:14:59 +0600 Subject: [PATCH 08/16] test(integration): add tests for cyclonedx and spdx --- integration/repo_test.go | 18 +++++ .../conda-environment-cyclonedx.json.golden | 80 +++++++++++++++++++ .../conda-environment-spdx.json.golden | 76 ++++++++++++++++++ .../repo/conda-environment/environment.yaml | 6 ++ 4 files changed, 180 insertions(+) create mode 100644 integration/testdata/conda-environment-cyclonedx.json.golden create mode 100644 integration/testdata/conda-environment-spdx.json.golden create mode 100644 integration/testdata/fixtures/repo/conda-environment/environment.yaml diff --git a/integration/repo_test.go b/integration/repo_test.go index ba11aa9ccb0f..4275e09d57cf 100644 --- a/integration/repo_test.go +++ b/integration/repo_test.go @@ -341,6 +341,15 @@ func TestRepository(t *testing.T) { }, golden: "testdata/conda-cyclonedx.json.golden", }, + { + name: "conda environment.yaml generating CycloneDX SBOM", + args: args{ + command: "fs", + format: "cyclonedx", + input: "testdata/fixtures/repo/conda-environment", + }, + golden: "testdata/conda-environment-cyclonedx.json.golden", + }, { name: "pom.xml generating CycloneDX SBOM (with vulnerabilities)", args: args{ @@ -360,6 +369,15 @@ func TestRepository(t *testing.T) { }, golden: "testdata/conda-spdx.json.golden", }, + { + name: "conda environment.yaml generating SPDX SBOM", + args: args{ + command: "fs", + format: "spdx-json", + input: "testdata/fixtures/repo/conda-environment", + }, + golden: "testdata/conda-environment-spdx.json.golden", + }, { name: "gomod with fs subcommand", args: args{ diff --git a/integration/testdata/conda-environment-cyclonedx.json.golden b/integration/testdata/conda-environment-cyclonedx.json.golden new file mode 100644 index 000000000000..e927b7594bfb --- /dev/null +++ b/integration/testdata/conda-environment-cyclonedx.json.golden @@ -0,0 +1,80 @@ +{ + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": "urn:uuid:3ff14136-e09f-4df9-80ea-000000000004", + "version": 1, + "metadata": { + "timestamp": "2021-08-25T12:20:30+00:00", + "tools": { + "components": [ + { + "type": "application", + "group": "aquasecurity", + "name": "trivy", + "version": "dev" + } + ] + }, + "component": { + "bom-ref": "3ff14136-e09f-4df9-80ea-000000000001", + "type": "application", + "name": "testdata/fixtures/repo/conda-environment", + "properties": [ + { + "name": "aquasecurity:trivy:SchemaVersion", + "value": "2" + } + ] + } + }, + "components": [ + { + "bom-ref": "3ff14136-e09f-4df9-80ea-000000000002", + "type": "application", + "name": "environment.yaml", + "properties": [ + { + "name": "aquasecurity:trivy:Class", + "value": "lang-pkgs" + }, + { + "name": "aquasecurity:trivy:Type", + "value": "conda-environment" + } + ] + }, + { + "bom-ref": "pkg:conda/bzip2@1.0.8", + "type": "library", + "name": "bzip2", + "version": "1.0.8", + "purl": "pkg:conda/bzip2@1.0.8", + "properties": [ + { + "name": "aquasecurity:trivy:PkgType", + "value": "conda-environment" + } + ] + } + ], + "dependencies": [ + { + "ref": "3ff14136-e09f-4df9-80ea-000000000001", + "dependsOn": [ + "3ff14136-e09f-4df9-80ea-000000000002" + ] + }, + { + "ref": "3ff14136-e09f-4df9-80ea-000000000002", + "dependsOn": [ + "pkg:conda/bzip2@1.0.8" + ] + }, + { + "ref": "pkg:conda/bzip2@1.0.8", + "dependsOn": [] + } + ], + "vulnerabilities": [] +} diff --git a/integration/testdata/conda-environment-spdx.json.golden b/integration/testdata/conda-environment-spdx.json.golden new file mode 100644 index 000000000000..c52b39d87e66 --- /dev/null +++ b/integration/testdata/conda-environment-spdx.json.golden @@ -0,0 +1,76 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "testdata/fixtures/repo/conda-environment", + "documentNamespace": "http://aquasecurity.github.io/trivy/filesystem/testdata/fixtures/repo/conda-environment-3ff14136-e09f-4df9-80ea-000000000004", + "creationInfo": { + "creators": [ + "Organization: aquasecurity", + "Tool: trivy-dev" + ], + "created": "2021-08-25T12:20:30Z" + }, + "packages": [ + { + "name": "environment.yaml", + "SPDXID": "SPDXRef-Application-202c31aa4648bfc2", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "attributionTexts": [ + "Class: lang-pkgs", + "Type: conda-environment" + ], + "primaryPackagePurpose": "APPLICATION" + }, + { + "name": "bzip2", + "SPDXID": "SPDXRef-Package-cba4fb2800432780", + "versionInfo": "1.0.8", + "supplier": "NOASSERTION", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "sourceInfo": "package found in: environment.yaml", + "licenseConcluded": "NONE", + "licenseDeclared": "NONE", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:conda/bzip2@1.0.8" + } + ], + "attributionTexts": [ + "PkgType: conda-environment" + ], + "primaryPackagePurpose": "LIBRARY" + }, + { + "name": "testdata/fixtures/repo/conda-environment", + "SPDXID": "SPDXRef-Filesystem-2702076b98393e0b", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "attributionTexts": [ + "SchemaVersion: 2" + ], + "primaryPackagePurpose": "SOURCE" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-Application-202c31aa4648bfc2", + "relatedSpdxElement": "SPDXRef-Package-cba4fb2800432780", + "relationshipType": "CONTAINS" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-Filesystem-2702076b98393e0b", + "relationshipType": "DESCRIBES" + }, + { + "spdxElementId": "SPDXRef-Filesystem-2702076b98393e0b", + "relatedSpdxElement": "SPDXRef-Application-202c31aa4648bfc2", + "relationshipType": "CONTAINS" + } + ] +} \ No newline at end of file diff --git a/integration/testdata/fixtures/repo/conda-environment/environment.yaml b/integration/testdata/fixtures/repo/conda-environment/environment.yaml new file mode 100644 index 000000000000..cf47d3632faf --- /dev/null +++ b/integration/testdata/fixtures/repo/conda-environment/environment.yaml @@ -0,0 +1,6 @@ +name: test-env +channels: + - defaults +dependencies: + - bzip2=1.0.8=h998d150_5 +prefix: /opt/conda/envs/test-env From 16a62e0da5f134967cb3de0377b0b8efe25cc7d9 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 15:38:32 +0600 Subject: [PATCH 09/16] ci(semantic-pr): add conda --- .github/workflows/semantic-pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semantic-pr.yaml b/.github/workflows/semantic-pr.yaml index 0b84c110abb5..f02ef758ae91 100644 --- a/.github/workflows/semantic-pr.yaml +++ b/.github/workflows/semantic-pr.yaml @@ -75,6 +75,7 @@ jobs: dart swift bitnami + conda os lang From 18da7b17b14da1f3a9b86c6d4e657d6dc499fda0 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 26 Apr 2024 15:58:40 +0600 Subject: [PATCH 10/16] fix linter error --- pkg/dependency/parser/conda/environment/parse.go | 10 ++++++---- .../analyzer/language/conda/environment/environment.go | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go index c8748019de7b..7cb4c20765fe 100644 --- a/pkg/dependency/parser/conda/environment/parse.go +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -1,13 +1,15 @@ package environment import ( + "sort" + "strings" + + "golang.org/x/xerrors" + "gopkg.in/yaml.v3" + "github.com/aquasecurity/trivy/pkg/dependency/types" "github.com/aquasecurity/trivy/pkg/log" xio "github.com/aquasecurity/trivy/pkg/x/io" - "golang.org/x/xerrors" - "gopkg.in/yaml.v3" - "sort" - "strings" ) type environment struct { diff --git a/pkg/fanal/analyzer/language/conda/environment/environment.go b/pkg/fanal/analyzer/language/conda/environment/environment.go index 3b700da96300..fbf8d8fc3451 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment.go @@ -2,13 +2,15 @@ package environment import ( "context" + "os" + "path/filepath" + + "golang.org/x/xerrors" + "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" - "golang.org/x/xerrors" - "os" - "path/filepath" ) func init() { From d5b0a5693f3f02e6b7a80f10cbee725b3e1a12b3 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Sat, 27 Apr 2024 09:28:34 +0600 Subject: [PATCH 11/16] test(integration): remove environment.yaml spdx test --- integration/repo_test.go | 9 --- .../conda-environment-spdx.json.golden | 76 ------------------- 2 files changed, 85 deletions(-) delete mode 100644 integration/testdata/conda-environment-spdx.json.golden diff --git a/integration/repo_test.go b/integration/repo_test.go index 4275e09d57cf..8d787104e63f 100644 --- a/integration/repo_test.go +++ b/integration/repo_test.go @@ -369,15 +369,6 @@ func TestRepository(t *testing.T) { }, golden: "testdata/conda-spdx.json.golden", }, - { - name: "conda environment.yaml generating SPDX SBOM", - args: args{ - command: "fs", - format: "spdx-json", - input: "testdata/fixtures/repo/conda-environment", - }, - golden: "testdata/conda-environment-spdx.json.golden", - }, { name: "gomod with fs subcommand", args: args{ diff --git a/integration/testdata/conda-environment-spdx.json.golden b/integration/testdata/conda-environment-spdx.json.golden deleted file mode 100644 index c52b39d87e66..000000000000 --- a/integration/testdata/conda-environment-spdx.json.golden +++ /dev/null @@ -1,76 +0,0 @@ -{ - "spdxVersion": "SPDX-2.3", - "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", - "name": "testdata/fixtures/repo/conda-environment", - "documentNamespace": "http://aquasecurity.github.io/trivy/filesystem/testdata/fixtures/repo/conda-environment-3ff14136-e09f-4df9-80ea-000000000004", - "creationInfo": { - "creators": [ - "Organization: aquasecurity", - "Tool: trivy-dev" - ], - "created": "2021-08-25T12:20:30Z" - }, - "packages": [ - { - "name": "environment.yaml", - "SPDXID": "SPDXRef-Application-202c31aa4648bfc2", - "downloadLocation": "NONE", - "filesAnalyzed": false, - "attributionTexts": [ - "Class: lang-pkgs", - "Type: conda-environment" - ], - "primaryPackagePurpose": "APPLICATION" - }, - { - "name": "bzip2", - "SPDXID": "SPDXRef-Package-cba4fb2800432780", - "versionInfo": "1.0.8", - "supplier": "NOASSERTION", - "downloadLocation": "NONE", - "filesAnalyzed": false, - "sourceInfo": "package found in: environment.yaml", - "licenseConcluded": "NONE", - "licenseDeclared": "NONE", - "externalRefs": [ - { - "referenceCategory": "PACKAGE-MANAGER", - "referenceType": "purl", - "referenceLocator": "pkg:conda/bzip2@1.0.8" - } - ], - "attributionTexts": [ - "PkgType: conda-environment" - ], - "primaryPackagePurpose": "LIBRARY" - }, - { - "name": "testdata/fixtures/repo/conda-environment", - "SPDXID": "SPDXRef-Filesystem-2702076b98393e0b", - "downloadLocation": "NONE", - "filesAnalyzed": false, - "attributionTexts": [ - "SchemaVersion: 2" - ], - "primaryPackagePurpose": "SOURCE" - } - ], - "relationships": [ - { - "spdxElementId": "SPDXRef-Application-202c31aa4648bfc2", - "relatedSpdxElement": "SPDXRef-Package-cba4fb2800432780", - "relationshipType": "CONTAINS" - }, - { - "spdxElementId": "SPDXRef-DOCUMENT", - "relatedSpdxElement": "SPDXRef-Filesystem-2702076b98393e0b", - "relationshipType": "DESCRIBES" - }, - { - "spdxElementId": "SPDXRef-Filesystem-2702076b98393e0b", - "relatedSpdxElement": "SPDXRef-Application-202c31aa4648bfc2", - "relationshipType": "CONTAINS" - } - ] -} \ No newline at end of file From 6fc4535e330945a008e361c5ff248e844374207b Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Sat, 27 Apr 2024 09:31:03 +0600 Subject: [PATCH 12/16] test(fanal): remove unneeded error check --- .../analyzer/language/conda/environment/environment_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/conda/environment/environment_test.go b/pkg/fanal/analyzer/language/conda/environment/environment_test.go index 02139200bd15..b9a8098b2879 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment_test.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment_test.go @@ -89,7 +89,6 @@ func Test_environmentAnalyzer_Analyze(t *testing.T) { }) if tt.wantErr != "" { - require.Error(t, err) require.ErrorContains(t, err, tt.wantErr) return } From 32b7ae23279b53ef769be4855350645006bb1308 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Sat, 27 Apr 2024 12:49:37 +0600 Subject: [PATCH 13/16] refactor(fanal): update logic to detect version of deps --- .../parser/conda/environment/parse.go | 50 +++++-- .../parser/conda/environment/parse_test.go | 133 ++++++++++++++++-- .../environment/testdata/happy-manually.yaml | 20 +++ .../conda/environment/testdata/happy.yaml | 2 - 4 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go index 7cb4c20765fe..4237e3aeb23f 100644 --- a/pkg/dependency/parser/conda/environment/parse.go +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -1,8 +1,10 @@ package environment import ( + "regexp" "sort" "strings" + "sync" "golang.org/x/xerrors" "gopkg.in/yaml.v3" @@ -23,14 +25,18 @@ type Dependency struct { type Parser struct { logger *log.Logger + once sync.Once } func NewParser() types.Parser { return &Parser{ logger: log.WithPrefix("conda"), + once: sync.Once{}, } } +var manuallyCreatedPkgRegexp = regexp.MustCompile(`(?P[A-Za-z0-9-_]+)( |>|<|=|!|$)`) + func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) { var env environment if err := yaml.NewDecoder(r).Decode(&env); err != nil { @@ -39,9 +45,10 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, var libs []types.Library for _, dep := range env.Dependencies { - lib, err := p.toLibrary(dep) - if err != nil { - return nil, nil, xerrors.Errorf("unable to parse dependency: %w", err) + lib := p.toLibrary(dep) + // Skip empty libs + if lib.Name == "" { + continue } libs = append(libs, lib) } @@ -50,14 +57,8 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, return libs, nil, nil } -func (p *Parser) toLibrary(dep Dependency) (types.Library, error) { - // Default format for files created using the `conda Export` command: `== - // e.g. `bzip2=1.0.8=h998d150_5` - // But it is also possible to set only the dependency name - ss := strings.Split(dep.Value, "=") - +func (p *Parser) toLibrary(dep Dependency) types.Library { lib := types.Library{ - Name: ss[0], Locations: types.Locations{ { StartLine: dep.Line, @@ -66,14 +67,33 @@ func (p *Parser) toLibrary(dep Dependency) (types.Library, error) { }, } - // Version can be omitted - if len(ss) == 1 { - p.logger.Warn("Unable to detect the version as it is not pinned", log.String("name", dep.Value)) - return lib, nil + // `Conda env export` command returns `=` format. + ss := strings.Split(dep.Value, "=") + // But `environment.yml` supports version range (for manually created files). + // cf. https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs + if len(ss) != 3 || strings.ContainsAny(dep.Value, "<>!*") || strings.Contains(dep.Value, "==") { + p.once.Do(func() { + p.logger.Warn("Unable to detect the versions of dependencies from `environment.yml` as they are not pinned. Use `conda env export` to pin versions.") + }) + + // Detect only name for manually created dependencies. + var name string + matches := manuallyCreatedPkgRegexp.FindStringSubmatch(dep.Value) + if matches != nil { + name = matches[manuallyCreatedPkgRegexp.SubexpIndex("name")] + } + if name == "" { + p.logger.Debug("Unable to parse dependency", log.String("dep", dep.Value)) + return types.Library{} + } + + lib.Name = name + return lib } + lib.Name = ss[0] lib.Version = ss[1] - return lib, nil + return lib } func (d *Dependency) UnmarshalYAML(node *yaml.Node) error { diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go index a1cefc43c3da..9a930917bb4e 100644 --- a/pkg/dependency/parser/conda/environment/parse_test.go +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -19,11 +19,12 @@ func TestParse(t *testing.T) { wantErr string }{ { - name: "happy path", + name: "happy path. Automatically created environment.yml file", input: "testdata/happy.yaml", want: []types.Library{ { - Name: "_libgcc_mutex", + Name: "blas", + Version: "1.0", Locations: types.Locations{ { StartLine: 5, @@ -32,17 +33,41 @@ func TestParse(t *testing.T) { }, }, { - Name: "_openmp_mutex", - Version: "5.1", + Name: "bzip2", + Version: "1.0.8", Locations: types.Locations{ { StartLine: 6, EndLine: 6, }, }, - }, { - Name: "blas", - Version: "1.0", + }, + }, + }, + { + name: "happy path. Manually created environment.yml file", + input: "testdata/happy-manually.yaml", + want: []types.Library{ + { + Name: "_openmp_mutex", + Locations: types.Locations{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "bzip2", + Locations: types.Locations{ + { + StartLine: 6, + EndLine: 6, + }, + }, + }, + { + Name: "ca-certificates", Locations: types.Locations{ { StartLine: 7, @@ -51,8 +76,7 @@ func TestParse(t *testing.T) { }, }, { - Name: "bzip2", - Version: "1.0.8", + Name: "ld_impl_linux-aarch64", Locations: types.Locations{ { StartLine: 8, @@ -60,6 +84,97 @@ func TestParse(t *testing.T) { }, }, }, + { + Name: "libblas", + Locations: types.Locations{ + { + StartLine: 9, + EndLine: 9, + }, + }, + }, + { + Name: "libcblas", + Locations: types.Locations{ + { + StartLine: 10, + EndLine: 10, + }, + }, + }, + { + Name: "libexpat", + Locations: types.Locations{ + { + StartLine: 11, + EndLine: 11, + }, + }, + }, + { + Name: "libffi", + Locations: types.Locations{ + { + StartLine: 12, + EndLine: 12, + }, + }, + }, + { + Name: "libgcc-ng", + Locations: types.Locations{ + { + StartLine: 13, + EndLine: 13, + }, + }, + }, + { + Name: "libgfortran-ng", + Locations: types.Locations{ + { + StartLine: 14, + EndLine: 14, + }, + }, + }, + { + Name: "libgfortran5", + Locations: types.Locations{ + { + StartLine: 15, + EndLine: 15, + }, + }, + }, + { + Name: "libgomp", + Locations: types.Locations{ + { + StartLine: 16, + EndLine: 16, + }, + }, + }, + { + Name: "liblapack", + Locations: types.Locations{ + { + StartLine: 17, + EndLine: 17, + }, + }, + }, + { + Name: "libnsl", + Version: "2.0.1", + Locations: types.Locations{ + { + StartLine: 18, + EndLine: 18, + }, + }, + }, }, }, { diff --git a/pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml b/pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml new file mode 100644 index 000000000000..685b951f34f4 --- /dev/null +++ b/pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml @@ -0,0 +1,20 @@ +name: test-env +channels: + - defaults +dependencies: + - _openmp_mutex + - bzip2=1 + - ca-certificates=2024.2 + - ld_impl_linux-aarch64=2.40.* + - libblas>=3.9 + - libcblas<=3.9.0=22_linuxaarch64_openblas + - libexpat==2.6.2 + - libffi==3.4.2=h3557bc0_5 + - libgcc-ng 13.2|13.3 + - libgfortran-ng >13.2.0,<=13.3 + - libgfortran5 =>13.2.0,<13.3|13.4 + - libgomp 13.2.0 hf8544c7_5 + - liblapack=3.9.*=22_linuxaarch64_openblas + - libnsl=2.0.1=h31becfc_0 + +prefix: /opt/conda/envs/test-env diff --git a/pkg/dependency/parser/conda/environment/testdata/happy.yaml b/pkg/dependency/parser/conda/environment/testdata/happy.yaml index 62cd7ff599bd..de7c3b63b015 100644 --- a/pkg/dependency/parser/conda/environment/testdata/happy.yaml +++ b/pkg/dependency/parser/conda/environment/testdata/happy.yaml @@ -2,8 +2,6 @@ name: test-env channels: - defaults dependencies: - - _libgcc_mutex - - _openmp_mutex=5.1 - blas=1.0=openblas - bzip2=1.0.8=h998d150_5 prefix: /opt/conda/envs/test-env From a3254f9edd8959d098311f911764f1713c062bd3 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Sat, 27 Apr 2024 12:53:21 +0600 Subject: [PATCH 14/16] refactor(fanal): use const for `environment.yaml` --- pkg/fanal/analyzer/language/conda/environment/environment.go | 2 +- .../analyzer/language/conda/environment/environment_test.go | 3 +-- pkg/fanal/types/const.go | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/fanal/analyzer/language/conda/environment/environment.go b/pkg/fanal/analyzer/language/conda/environment/environment.go index fbf8d8fc3451..ee4dfbd7de88 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment.go @@ -29,7 +29,7 @@ func (a environmentAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisI return res, nil } func (a environmentAnalyzer) Required(filePath string, _ os.FileInfo) bool { - return filepath.Base(filePath) == "environment.yaml" || filepath.Base(filePath) == "environment.yml" + return filepath.Base(filePath) == types.CondaEnvYml || filepath.Base(filePath) == types.CondaEnvYaml } func (a environmentAnalyzer) Type() analyzer.Type { diff --git a/pkg/fanal/analyzer/language/conda/environment/environment_test.go b/pkg/fanal/analyzer/language/conda/environment/environment_test.go index b9a8098b2879..2a35aea316bb 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment_test.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment_test.go @@ -36,8 +36,7 @@ func Test_environmentAnalyzer_Analyze(t *testing.T) { }, }, { - Name: "_openmp_mutex", - Version: "5.1", + Name: "_openmp_mutex", Locations: []types.Location{ { StartLine: 6, diff --git a/pkg/fanal/types/const.go b/pkg/fanal/types/const.go index 52769674539b..56f56036a590 100644 --- a/pkg/fanal/types/const.go +++ b/pkg/fanal/types/const.go @@ -140,4 +140,7 @@ const ( PubSpecLock = "pubspec.lock" MixLock = "mix.lock" + + CondaEnvYaml = "environment.yaml" + CondaEnvYml = "environment.yml" ) From 3425472f382148077b7c5870d7c43dd037b5b1a5 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Sat, 27 Apr 2024 13:23:06 +0600 Subject: [PATCH 15/16] docs: add conda page --- docs/docs/coverage/os/conda.md | 36 ++++++++++++++++++++++++++++++++++ docs/docs/coverage/os/index.md | 35 +++++++++++++++++---------------- mkdocs.yml | 1 + 3 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 docs/docs/coverage/os/conda.md diff --git a/docs/docs/coverage/os/conda.md b/docs/docs/coverage/os/conda.md new file mode 100644 index 000000000000..79a49194fd66 --- /dev/null +++ b/docs/docs/coverage/os/conda.md @@ -0,0 +1,36 @@ +# Conda + +Trivy supports the following scanners for Conda packages. + +| Scanner | Supported | +|:-------------:|:---------:| +| SBOM | ✓ | +| Vulnerability | - | +| License | ✓[^1] | + + +## SBOM +Trivy detects packages that have been installed with `Conda`. + + +### `.json` +Trivy parses `/envs//conda-meta/.json` files to find the version and license for the dependencies installed in your env. + +### `environment.yml`[^2] +Trivy supports parsing [environment.yml][environment.yml][^2] files to find dependency list. + +!!! note + License detection is currently not supported. + +`environment.yml`[^2] files supports [version range][env-version-range]. We can't be sure about versions for these dependencies. +Therefore, you need to use `conda env export` command to get dependency list in `Conda` default format before scanning `environment.yml`[^2] file. + +!!! note + For dependencies in a non-Conda format, Trivy doesn't include a version of them. + + +[^1]: License detection is only supported for `.json` files +[^2]: Trivy supports both `yaml` and `yml` extensions. + +[environment.yml]: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#sharing-an-environment +[env-version-range]: https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs diff --git a/docs/docs/coverage/os/index.md b/docs/docs/coverage/os/index.md index e04a452fc4d3..3294557d6a7c 100644 --- a/docs/docs/coverage/os/index.md +++ b/docs/docs/coverage/os/index.md @@ -9,23 +9,24 @@ Trivy supports operating systems for ## Supported OS -| OS | Supported Versions | Package Managers | -|-----------------------------------------------|-------------------------------------|------------------| -| [Alpine Linux](alpine.md) | 2.2 - 2.7, 3.0 - 3.19, edge | apk | -| [Wolfi Linux](wolfi.md) | (n/a) | apk | -| [Chainguard](chainguard.md) | (n/a) | apk | -| [Red Hat Enterprise Linux](rhel.md) | 6, 7, 8 | dnf/yum/rpm | -| [CentOS](centos.md)[^1] | 6, 7, 8 | dnf/yum/rpm | -| [AlmaLinux](alma.md) | 8, 9 | dnf/yum/rpm | -| [Rocky Linux](rocky.md) | 8, 9 | dnf/yum/rpm | -| [Oracle Linux](oracle.md) | 5, 6, 7, 8 | dnf/yum/rpm | -| [CBL-Mariner](cbl-mariner.md) | 1.0, 2.0 | dnf/yum/rpm | -| [Amazon Linux](amazon.md) | 1, 2, 2023 | dnf/yum/rpm | -| [openSUSE Leap](suse.md) | 42, 15 | zypper/rpm | -| [SUSE Enterprise Linux](suse.md) | 11, 12, 15 | zypper/rpm | -| [Photon OS](photon.md) | 1.0, 2.0, 3.0, 4.0 | tndf/yum/rpm | -| [Debian GNU/Linux](debian.md) | 7, 8, 9, 10, 11, 12 | apt/dpkg | -| [Ubuntu](ubuntu.md) | All versions supported by Canonical | apt/dpkg | +| OS | Supported Versions | Package Managers | +|--------------------------------------|-------------------------------------|------------------| +| [Alpine Linux](alpine.md) | 2.2 - 2.7, 3.0 - 3.19, edge | apk | +| [Wolfi Linux](wolfi.md) | (n/a) | apk | +| [Chainguard](chainguard.md) | (n/a) | apk | +| [Red Hat Enterprise Linux](rhel.md) | 6, 7, 8 | dnf/yum/rpm | +| [CentOS](centos.md)[^1] | 6, 7, 8 | dnf/yum/rpm | +| [AlmaLinux](alma.md) | 8, 9 | dnf/yum/rpm | +| [Rocky Linux](rocky.md) | 8, 9 | dnf/yum/rpm | +| [Oracle Linux](oracle.md) | 5, 6, 7, 8 | dnf/yum/rpm | +| [CBL-Mariner](cbl-mariner.md) | 1.0, 2.0 | dnf/yum/rpm | +| [Amazon Linux](amazon.md) | 1, 2, 2023 | dnf/yum/rpm | +| [openSUSE Leap](suse.md) | 42, 15 | zypper/rpm | +| [SUSE Enterprise Linux](suse.md) | 11, 12, 15 | zypper/rpm | +| [Photon OS](photon.md) | 1.0, 2.0, 3.0, 4.0 | tndf/yum/rpm | +| [Debian GNU/Linux](debian.md) | 7, 8, 9, 10, 11, 12 | apt/dpkg | +| [Ubuntu](ubuntu.md) | All versions supported by Canonical | apt/dpkg | +| [OSs with installed Conda](conda.md) | - | conda | ## Supported container images diff --git a/mkdocs.yml b/mkdocs.yml index f85fd7a8f209..75aff50f9fe2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,6 +77,7 @@ nav: - CBL-Mariner: docs/coverage/os/cbl-mariner.md - CentOS: docs/coverage/os/centos.md - Chainguard: docs/coverage/os/chainguard.md + - Conda: docs/coverage/os/conda.md - Debian: docs/coverage/os/debian.md - Oracle Linux: docs/coverage/os/oracle.md - Photon OS: docs/coverage/os/photon.md From 3b49a17e0031dd48aee59b2811afa79ac0e6e759 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 29 Apr 2024 12:11:47 +0400 Subject: [PATCH 16/16] feat: parse more versions Signed-off-by: knqyf263 --- .../parser/conda/environment/parse.go | 60 +++++++++---------- .../parser/conda/environment/parse_test.go | 43 +++++-------- .../environment/testdata/happy-manually.yaml | 20 ------- .../conda/environment/testdata/happy.yaml | 14 +++++ .../conda/environment/environment_test.go | 6 +- 5 files changed, 64 insertions(+), 79 deletions(-) delete mode 100644 pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go index 4237e3aeb23f..8a4418699f2f 100644 --- a/pkg/dependency/parser/conda/environment/parse.go +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -1,7 +1,6 @@ package environment import ( - "regexp" "sort" "strings" "sync" @@ -9,6 +8,7 @@ import ( "golang.org/x/xerrors" "gopkg.in/yaml.v3" + "github.com/aquasecurity/go-version/pkg/version" "github.com/aquasecurity/trivy/pkg/dependency/types" "github.com/aquasecurity/trivy/pkg/log" xio "github.com/aquasecurity/trivy/pkg/x/io" @@ -35,8 +35,6 @@ func NewParser() types.Parser { } } -var manuallyCreatedPkgRegexp = regexp.MustCompile(`(?P[A-Za-z0-9-_]+)( |>|<|=|!|$)`) - func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) { var env environment if err := yaml.NewDecoder(r).Decode(&env); err != nil { @@ -58,7 +56,15 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, } func (p *Parser) toLibrary(dep Dependency) types.Library { - lib := types.Library{ + name, ver := p.parseDependency(dep.Value) + if ver == "" { + p.once.Do(func() { + p.logger.Warn("Unable to detect the dependency versions from `environment.yml` as those versions are not pinned. Use `conda env export` to pin versions.") + }) + } + return types.Library{ + Name: name, + Version: ver, Locations: types.Locations{ { StartLine: dep.Line, @@ -66,34 +72,28 @@ func (p *Parser) toLibrary(dep Dependency) types.Library { }, }, } +} - // `Conda env export` command returns `=` format. - ss := strings.Split(dep.Value, "=") - // But `environment.yml` supports version range (for manually created files). - // cf. https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs - if len(ss) != 3 || strings.ContainsAny(dep.Value, "<>!*") || strings.Contains(dep.Value, "==") { - p.once.Do(func() { - p.logger.Warn("Unable to detect the versions of dependencies from `environment.yml` as they are not pinned. Use `conda env export` to pin versions.") - }) - - // Detect only name for manually created dependencies. - var name string - matches := manuallyCreatedPkgRegexp.FindStringSubmatch(dep.Value) - if matches != nil { - name = matches[manuallyCreatedPkgRegexp.SubexpIndex("name")] - } - if name == "" { - p.logger.Debug("Unable to parse dependency", log.String("dep", dep.Value)) - return types.Library{} - } - - lib.Name = name - return lib +// parseDependency parses the dependency line and returns the name and the pinned version. +// The version range is not supported. It parses only the pinned version. +// e.g. +// - numpy 1.8.1 +// - numpy ==1.8.1 +// - numpy 1.8.1 py27_0 +// - numpy=1.8.1=py27_0 +// +// cf. https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs +func (*Parser) parseDependency(line string) (string, string) { + line = strings.NewReplacer(">", " >", "<", " <", "=", " ").Replace(line) + parts := strings.Fields(line) + name := parts[0] + if len(parts) == 1 { + return name, "" } - - lib.Name = ss[0] - lib.Version = ss[1] - return lib + if _, err := version.Parse(parts[1]); err != nil { + return name, "" + } + return name, parts[1] } func (d *Dependency) UnmarshalYAML(node *yaml.Node) error { diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go index 9a930917bb4e..f68736947119 100644 --- a/pkg/dependency/parser/conda/environment/parse_test.go +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -19,22 +19,11 @@ func TestParse(t *testing.T) { wantErr string }{ { - name: "happy path. Automatically created environment.yml file", + name: "happy path", input: "testdata/happy.yaml", want: []types.Library{ { - Name: "blas", - Version: "1.0", - Locations: types.Locations{ - { - StartLine: 5, - EndLine: 5, - }, - }, - }, - { - Name: "bzip2", - Version: "1.0.8", + Name: "_openmp_mutex", Locations: types.Locations{ { StartLine: 6, @@ -42,14 +31,9 @@ func TestParse(t *testing.T) { }, }, }, - }, - }, - { - name: "happy path. Manually created environment.yml file", - input: "testdata/happy-manually.yaml", - want: []types.Library{ { - Name: "_openmp_mutex", + Name: "blas", + Version: "1.0", Locations: types.Locations{ { StartLine: 5, @@ -58,16 +42,18 @@ func TestParse(t *testing.T) { }, }, { - Name: "bzip2", + Name: "bzip2", + Version: "1.0.8", Locations: types.Locations{ { - StartLine: 6, - EndLine: 6, + StartLine: 19, + EndLine: 19, }, }, }, { - Name: "ca-certificates", + Name: "ca-certificates", + Version: "2024.2", Locations: types.Locations{ { StartLine: 7, @@ -103,7 +89,8 @@ func TestParse(t *testing.T) { }, }, { - Name: "libexpat", + Name: "libexpat", + Version: "2.6.2", Locations: types.Locations{ { StartLine: 11, @@ -112,7 +99,8 @@ func TestParse(t *testing.T) { }, }, { - Name: "libffi", + Name: "libffi", + Version: "3.4.2", Locations: types.Locations{ { StartLine: 12, @@ -148,7 +136,8 @@ func TestParse(t *testing.T) { }, }, { - Name: "libgomp", + Name: "libgomp", + Version: "13.2.0", Locations: types.Locations{ { StartLine: 16, diff --git a/pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml b/pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml deleted file mode 100644 index 685b951f34f4..000000000000 --- a/pkg/dependency/parser/conda/environment/testdata/happy-manually.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: test-env -channels: - - defaults -dependencies: - - _openmp_mutex - - bzip2=1 - - ca-certificates=2024.2 - - ld_impl_linux-aarch64=2.40.* - - libblas>=3.9 - - libcblas<=3.9.0=22_linuxaarch64_openblas - - libexpat==2.6.2 - - libffi==3.4.2=h3557bc0_5 - - libgcc-ng 13.2|13.3 - - libgfortran-ng >13.2.0,<=13.3 - - libgfortran5 =>13.2.0,<13.3|13.4 - - libgomp 13.2.0 hf8544c7_5 - - liblapack=3.9.*=22_linuxaarch64_openblas - - libnsl=2.0.1=h31becfc_0 - -prefix: /opt/conda/envs/test-env diff --git a/pkg/dependency/parser/conda/environment/testdata/happy.yaml b/pkg/dependency/parser/conda/environment/testdata/happy.yaml index de7c3b63b015..f36e8bf990eb 100644 --- a/pkg/dependency/parser/conda/environment/testdata/happy.yaml +++ b/pkg/dependency/parser/conda/environment/testdata/happy.yaml @@ -3,5 +3,19 @@ channels: - defaults dependencies: - blas=1.0=openblas + - _openmp_mutex + - ca-certificates=2024.2 + - ld_impl_linux-aarch64=2.40.* + - libblas>=3.9 + - libcblas<=3.9.0=22_linuxaarch64_openblas + - libexpat==2.6.2 + - libffi==3.4.2=h3557bc0_5 + - libgcc-ng 13.2|13.3 + - libgfortran-ng >13.2.0,<=13.3 + - libgfortran5 =>13.2.0,<13.3|13.4 + - libgomp 13.2.0 hf8544c7_5 + - liblapack=3.9.*=22_linuxaarch64_openblas + - libnsl=2.0.1=h31becfc_0 - bzip2=1.0.8=h998d150_5 + prefix: /opt/conda/envs/test-env diff --git a/pkg/fanal/analyzer/language/conda/environment/environment_test.go b/pkg/fanal/analyzer/language/conda/environment/environment_test.go index 2a35aea316bb..d511ac3e50a1 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment_test.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment_test.go @@ -36,14 +36,16 @@ func Test_environmentAnalyzer_Analyze(t *testing.T) { }, }, { - Name: "_openmp_mutex", + Name: "_openmp_mutex", + Version: "5.1", Locations: []types.Location{ { StartLine: 6, EndLine: 6, }, }, - }, { + }, + { Name: "blas", Version: "1.0", Locations: []types.Location{