From 2bc54ad2752aba5de4380cb92c13b09c0abefd73 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Mon, 20 May 2024 22:55:36 +0600 Subject: [PATCH] feat(python): add line number support for `requirement.txt` files (#6729) --- docs/docs/coverage/language/python.md | 4 +- integration/testdata/pip.json.golden | 74 +++-- pkg/dependency/parser/python/pip/parse.go | 8 + .../parser/python/pip/parse_testcase.go | 264 ++++++++++++++++-- .../analyzer/language/python/pip/pip_test.go | 18 ++ pkg/fanal/artifact/local/fs_test.go | 24 +- 6 files changed, 344 insertions(+), 48 deletions(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index eaed8f1d4b7d..ce792842b978 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -23,9 +23,9 @@ The following table provides an outline of the features Trivy offers. | Package manager | File | Transitive dependencies | Dev dependencies | [Dependency graph][dependency-graph] | Position | |-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:| -| pip | requirements.txt | - | Include | - | - | +| pip | requirements.txt | - | Include | - | ✓ | | Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | -| Poetry | poetry.lock | ✓ | Exclude | ✓ | | +| Poetry | poetry.lock | ✓ | Exclude | ✓ | - | | Packaging | Dependency graph | diff --git a/integration/testdata/pip.json.golden b/integration/testdata/pip.json.golden index cc715f05357d..4171d6b91df2 100644 --- a/integration/testdata/pip.json.golden +++ b/integration/testdata/pip.json.golden @@ -25,64 +25,106 @@ "Name": "Flask", "Identifier": { "PURL": "pkg:pypi/flask@2.0.0", - "UID": "301ccf5fd90d6082" + "UID": "8b02ba2c070d72c6" }, "Version": "2.0.0", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 2, + "EndLine": 2 + } + ] }, { "Name": "Jinja2", "Identifier": { "PURL": "pkg:pypi/jinja2@3.0.0", - "UID": "212193e1595e68cc" + "UID": "476df0c1e49c8f99" }, "Version": "3.0.0", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 4, + "EndLine": 4 + } + ] }, { "Name": "Werkzeug", "Identifier": { "PURL": "pkg:pypi/werkzeug@0.11", - "UID": "56b919b561299a48" + "UID": "4163de19df046f49" }, "Version": "0.11", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 6, + "EndLine": 6 + } + ] }, { "Name": "click", "Identifier": { "PURL": "pkg:pypi/click@8.0.0", - "UID": "d58cb56b4e8b1ffd" + "UID": "71e4c8ef31456bf" }, "Version": "8.0.0", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 1, + "EndLine": 1 + } + ] }, { "Name": "itsdangerous", "Identifier": { "PURL": "pkg:pypi/itsdangerous@2.0.0", - "UID": "9bf39d440e409733" + "UID": "389c7cbc34cb6b32" }, "Version": "2.0.0", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 3, + "EndLine": 3 + } + ] }, { "Name": "oauth2-client", "Identifier": { "PURL": "pkg:pypi/oauth2-client@4.0.0", - "UID": "ffc67df5ef686f77" + "UID": "c63f60db796a16ed" }, "Version": "4.0.0", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 7, + "EndLine": 7 + } + ] }, { "Name": "python-gitlab", "Identifier": { "PURL": "pkg:pypi/python-gitlab@2.0.0", - "UID": "f9cbb9736717c4d4" + "UID": "ccad39abab737d13" }, "Version": "2.0.0", - "Layer": {} + "Layer": {}, + "Locations": [ + { + "StartLine": 8, + "EndLine": 8 + } + ] } ], "Vulnerabilities": [ @@ -91,7 +133,7 @@ "PkgName": "Werkzeug", "PkgIdentifier": { "PURL": "pkg:pypi/werkzeug@0.11", - "UID": "56b919b561299a48" + "UID": "4163de19df046f49" }, "InstalledVersion": "0.11", "FixedVersion": "0.15.3", @@ -148,7 +190,7 @@ "PkgName": "Werkzeug", "PkgIdentifier": { "PURL": "pkg:pypi/werkzeug@0.11", - "UID": "56b919b561299a48" + "UID": "4163de19df046f49" }, "InstalledVersion": "0.11", "FixedVersion": "0.11.6", diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index 00eee4349b0b..a849eb0cbea2 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -37,7 +37,9 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc scanner := bufio.NewScanner(decodedReader) var pkgs []ftypes.Package + var lineNumber int for scanner.Scan() { + lineNumber++ line := scanner.Text() line = strings.ReplaceAll(line, " ", "") line = strings.ReplaceAll(line, `\`, "") @@ -52,6 +54,12 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc pkgs = append(pkgs, ftypes.Package{ Name: s[0], Version: s[1], + Locations: []ftypes.Location{ + { + StartLine: lineNumber, + EndLine: lineNumber, + }, + }, }) } if err := scanner.Err(); err != nil { diff --git a/pkg/dependency/parser/python/pip/parse_testcase.go b/pkg/dependency/parser/python/pip/parse_testcase.go index dc119c3ba054..e8192ee1775d 100644 --- a/pkg/dependency/parser/python/pip/parse_testcase.go +++ b/pkg/dependency/parser/python/pip/parse_testcase.go @@ -4,53 +4,269 @@ import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" var ( requirementsFlask = []ftypes.Package{ - {Name: "click", Version: "8.0.0"}, - {Name: "Flask", Version: "2.0.0"}, - {Name: "itsdangerous", Version: "2.0.0"}, - {Name: "Jinja2", Version: "3.0.0"}, - {Name: "MarkupSafe", Version: "2.0.0"}, - {Name: "Werkzeug", Version: "2.0.0"}, + { + Name: "click", + Version: "8.0.0", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, + { + Name: "Flask", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, + }, + { + Name: "itsdangerous", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 3, + EndLine: 3, + }, + }, + }, + { + Name: "Jinja2", + Version: "3.0.0", + Locations: []ftypes.Location{ + { + StartLine: 4, + EndLine: 4, + }, + }, + }, + { + Name: "MarkupSafe", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "Werkzeug", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 6, + EndLine: 6, + }, + }, + }, } requirementsComments = []ftypes.Package{ - {Name: "click", Version: "8.0.0"}, - {Name: "Flask", Version: "2.0.0"}, - {Name: "Jinja2", Version: "3.0.0"}, - {Name: "MarkupSafe", Version: "2.0.0"}, + { + Name: "click", + Version: "8.0.0", + Locations: []ftypes.Location{ + { + StartLine: 4, + EndLine: 4, + }, + }, + }, + { + Name: "Flask", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "Jinja2", + Version: "3.0.0", + Locations: []ftypes.Location{ + { + StartLine: 6, + EndLine: 6, + }, + }, + }, + { + Name: "MarkupSafe", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 7, + EndLine: 7, + }, + }, + }, } requirementsSpaces = []ftypes.Package{ - {Name: "click", Version: "8.0.0"}, - {Name: "Flask", Version: "2.0.0"}, - {Name: "itsdangerous", Version: "2.0.0"}, - {Name: "Jinja2", Version: "3.0.0"}, + { + Name: "click", + Version: "8.0.0", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, + { + Name: "Flask", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, + }, + { + Name: "itsdangerous", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 3, + EndLine: 3, + }, + }, + }, + { + Name: "Jinja2", + Version: "3.0.0", + Locations: []ftypes.Location{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, } requirementsNoVersion = []ftypes.Package{ - {Name: "Flask", Version: "2.0.0"}, + { + Name: "Flask", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, } requirementsOperator = []ftypes.Package{ - {Name: "Django", Version: "2.3.4"}, - {Name: "SomeProject", Version: "5.4"}, + { + Name: "Django", + Version: "2.3.4", + Locations: []ftypes.Location{ + { + StartLine: 4, + EndLine: 4, + }, + }, + }, + { + Name: "SomeProject", + Version: "5.4", + Locations: []ftypes.Location{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, } requirementsHash = []ftypes.Package{ - {Name: "FooProject", Version: "1.2"}, - {Name: "Jinja2", Version: "3.0.0"}, + { + Name: "FooProject", + Version: "1.2", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, + { + Name: "Jinja2", + Version: "3.0.0", + Locations: []ftypes.Location{ + { + StartLine: 4, + EndLine: 4, + }, + }, + }, } requirementsHyphens = []ftypes.Package{ - {Name: "oauth2-client", Version: "4.0.0"}, - {Name: "python-gitlab", Version: "2.0.0"}, + { + Name: "oauth2-client", + Version: "4.0.0", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, + { + Name: "python-gitlab", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, + }, } requirementsExtras = []ftypes.Package{ - {Name: "pyjwt", Version: "2.1.0"}, - {Name: "celery", Version: "4.4.7"}, + { + Name: "pyjwt", + Version: "2.1.0", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, + { + Name: "celery", + Version: "4.4.7", + Locations: []ftypes.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, + }, } requirementsUtf16le = []ftypes.Package{ - {Name: "attrs", Version: "20.3.0"}, + { + Name: "attrs", + Version: "20.3.0", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, } ) diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index c97714506a27..62fd13953f9b 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -31,14 +31,32 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { { Name: "click", Version: "8.0.0", + Locations: []types.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, }, { Name: "Flask", Version: "2.0.0", + Locations: []types.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, }, { Name: "itsdangerous", Version: "2.0.0", + Locations: []types.Location{ + { + StartLine: 3, + EndLine: 3, + }, + }, }, }, }, diff --git a/pkg/fanal/artifact/local/fs_test.go b/pkg/fanal/artifact/local/fs_test.go index 3bbb9d2c3b79..707818c3cdf2 100644 --- a/pkg/fanal/artifact/local/fs_test.go +++ b/pkg/fanal/artifact/local/fs_test.go @@ -174,7 +174,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:9e999fcf6fd571e175601fb7cc0da28f0d7960e26eab67dad93152e0bebf21ca", + BlobID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, Applications: []types.Application{ @@ -185,6 +185,12 @@ func TestArtifact_Inspect(t *testing.T) { { Name: "Flask", Version: "2.0.0", + Locations: []types.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, }, }, }, @@ -196,9 +202,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "testdata/requirements.txt", Type: artifact.TypeFilesystem, - ID: "sha256:9e999fcf6fd571e175601fb7cc0da28f0d7960e26eab67dad93152e0bebf21ca", + ID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", BlobIDs: []string{ - "sha256:9e999fcf6fd571e175601fb7cc0da28f0d7960e26eab67dad93152e0bebf21ca", + "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", }, }, }, @@ -209,7 +215,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:9e999fcf6fd571e175601fb7cc0da28f0d7960e26eab67dad93152e0bebf21ca", + BlobID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, Applications: []types.Application{ @@ -220,6 +226,12 @@ func TestArtifact_Inspect(t *testing.T) { { Name: "Flask", Version: "2.0.0", + Locations: []types.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, }, }, }, @@ -231,9 +243,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "testdata/requirements.txt", Type: artifact.TypeFilesystem, - ID: "sha256:9e999fcf6fd571e175601fb7cc0da28f0d7960e26eab67dad93152e0bebf21ca", + ID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", BlobIDs: []string{ - "sha256:9e999fcf6fd571e175601fb7cc0da28f0d7960e26eab67dad93152e0bebf21ca", + "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", }, }, },