From ca67153d8b4f6a9b35e83ad09291e710df85055b Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Tue, 21 May 2024 06:27:48 +0000 Subject: [PATCH 01/32] Initial commit of a very bare-bones linter ``` go run cmd/osv/main.go record lint test_data/nointroduced-CVE-2023-41045.json ``` Signed-off-by: Andrew Pollock --- go.work | 3 + go.work.sum | 2 + tools/osv-linter/cmd/osv/main.go | 46 ++++++++++ tools/osv-linter/go.mod | 16 ++++ tools/osv-linter/go.sum | 14 +++ tools/osv-linter/internal/checks/checks.go | 43 +++++++++ tools/osv-linter/internal/checks/ranges.go | 25 ++++++ tools/osv-linter/internal/linter.go | 87 +++++++++++++++++++ .../nointroduced-CVE-2023-41045.json | 51 +++++++++++ 9 files changed, 287 insertions(+) create mode 100644 go.work create mode 100644 go.work.sum create mode 100644 tools/osv-linter/cmd/osv/main.go create mode 100644 tools/osv-linter/go.mod create mode 100644 tools/osv-linter/go.sum create mode 100644 tools/osv-linter/internal/checks/checks.go create mode 100644 tools/osv-linter/internal/checks/ranges.go create mode 100644 tools/osv-linter/internal/linter.go create mode 100644 tools/osv-linter/test_data/nointroduced-CVE-2023-41045.json diff --git a/go.work b/go.work new file mode 100644 index 00000000..5d2774fa --- /dev/null +++ b/go.work @@ -0,0 +1,3 @@ +go 1.21.10 + +use ./tools/osv-linter diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 00000000..708d15c5 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,2 @@ +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/osv-linter/cmd/osv/main.go b/tools/osv-linter/cmd/osv/main.go new file mode 100644 index 00000000..fad4223f --- /dev/null +++ b/tools/osv-linter/cmd/osv/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "log" + "os" + + "github.com/ossf/osv-schema/linter/internal" + "github.com/urfave/cli/v2" +) + +func main() { + app := &cli.App{ + Name: "osv", + Usage: "OSV general purpose tool", + Commands: []*cli.Command{ + { + Name: "record", + Usage: "operations on OSV records", + Subcommands: []*cli.Command{ + { + Name: "lint", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "collection", + Value: "osv.dev", + Usage: "check collection to use (use 'list' to see)", + }, + &cli.StringFlag{ + Name: "check", + Value: "", + Usage: "explicitly run a single check (use 'list' to see)", + }, + }, + Aliases: []string{"check"}, + Usage: "check OSV records for correctness", + Action: internal.LintCommand, + }, + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} \ No newline at end of file diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod new file mode 100644 index 00000000..14563a6b --- /dev/null +++ b/tools/osv-linter/go.mod @@ -0,0 +1,16 @@ +module github.com/ossf/osv-schema/linter + +go 1.21.10 + +require ( + github.com/tidwall/gjson v1.17.1 + github.com/urfave/cli/v2 v2.27.2 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect +) diff --git a/tools/osv-linter/go.sum b/tools/osv-linter/go.sum new file mode 100644 index 00000000..f2454fdd --- /dev/null +++ b/tools/osv-linter/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go new file mode 100644 index 00000000..a6d45f21 --- /dev/null +++ b/tools/osv-linter/internal/checks/checks.go @@ -0,0 +1,43 @@ +package checks + +import "github.com/tidwall/gjson" + +type CheckCode string + +type CheckError struct { + Text string +} + +type Check struct { + Code CheckCode + Name string + Description string + Check func(*gjson.Result) []CheckError +} + +type CheckCollection struct { + Name string + Description string + Checks []*Check +} + +var CheckIntroducedEventExists = &Check{ + Code: "R0001", + Name: "introduced-event-exists", + Description: "every range has an introduced event", + Check: RangeHasIntroducedEvent, +} + +var AllChecks = map[string]*Check{ + "introduced-event-exists": CheckIntroducedEventExists, +} + +var CheckCollections = map[string]CheckCollection{ + "osv.dev": { + Name: "osv.dev", + Description: "the checks OSV.dev considers necessary for a high quality record", + Checks: []*Check{ + CheckIntroducedEventExists, + }, + }, +} diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go new file mode 100644 index 00000000..24d40265 --- /dev/null +++ b/tools/osv-linter/internal/checks/ranges.go @@ -0,0 +1,25 @@ +package checks + +import ( + "fmt" + + "github.com/tidwall/gjson" +) + +func RangeHasIntroducedEvent(json *gjson.Result) []CheckError { + result := json.Get(`affected.#.ranges.#.events`) + + findings := []CheckError{} + + result.ForEach(func(key, value gjson.Result) bool { + if !value.Get("introduced").Exists() { + findings = append(findings, CheckError{Text: fmt.Sprintf("Error: Missing 'introduced' object in event at index %s", key)}) + } + return true // Continue iteration. + }) + + if len(findings) != 0 { + return findings + } + return nil +} diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go new file mode 100644 index 00000000..7d45a9a1 --- /dev/null +++ b/tools/osv-linter/internal/linter.go @@ -0,0 +1,87 @@ +package internal + +import ( + "errors" + "fmt" + "log" + "os" + + "github.com/tidwall/gjson" + + "github.com/urfave/cli/v2" + + "github.com/ossf/osv-schema/linter/internal/checks" +) + +func LintCommand(cCtx *cli.Context) error { + if cCtx.String("collection") == "list" { + fmt.Printf("Available check collections:\n\n") + for _, collection := range checks.CheckCollections { + fmt.Printf("%s: %s\n", collection.Name, collection.Description) + } + return nil + } + + if cCtx.String("check") == "list" { + fmt.Printf("Available checks:\n\n") + for _, check := range checks.AllChecks { + fmt.Printf("%s: %s\n", check.Name, check.Description) + } + return nil + } + + if cCtx.NArg() == 0 { + return errors.New("nothing to check") + } + + for _, fileToCheck := range cCtx.Args().Slice() { + + // Check file exists. + recordBytes, err := os.ReadFile(fileToCheck) + if err != nil { + log.Printf("%v, skipping", err) + continue + } + + // Parse file into JSON + if !gjson.ValidBytes(recordBytes) { + log.Printf("%q: invalid JSON", fileToCheck) + } + + record := gjson.ParseBytes(recordBytes) + + if cCtx.String("check") != "" { + fmt.Printf("Running %q check on %q\n", cCtx.String("check"), fileToCheck) + // Check the requested check exists. + if _, ok := checks.AllChecks[cCtx.String("check")]; !ok { + return fmt.Errorf("%q is not a valid check", cCtx.String("check")) + } + // Run just the requested check. + check := checks.AllChecks[cCtx.String("check")] + // TODO: store in a per-file map so a per-file summary can be produced. + result := check.Check(&record) + if result != nil { + log.Printf("%q: %q: %#v", fileToCheck, cCtx.String("check"), result) + } + continue + } + + if cCtx.String("collection") != "" { + fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) + // Check the requested check collection exists. + if _, ok := checks.CheckCollections[cCtx.String("collection")]; !ok { + return fmt.Errorf("%q is not a valid check collection", cCtx.String("collection")) + } + // Run all checks in collection + for _, check := range checks.CheckCollections[cCtx.String("collection")].Checks { + // TODO: store in a per-file per-check map so a per-file summary can be produced. + result := check.Check(&record) + if result != nil { + log.Printf("%q: %q: %#v", fileToCheck, cCtx.String("check"), result) + } + } + continue + } + } + return nil +} diff --git a/tools/osv-linter/test_data/nointroduced-CVE-2023-41045.json b/tools/osv-linter/test_data/nointroduced-CVE-2023-41045.json new file mode 100644 index 00000000..6311f206 --- /dev/null +++ b/tools/osv-linter/test_data/nointroduced-CVE-2023-41045.json @@ -0,0 +1,51 @@ +{ + "id": "CVE-2023-41045", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:L/A:N" + } + ], + "details": "Graylog is a free and open log management platform. Graylog makes use of only one single source port for DNS queries. Graylog binds a single socket for outgoing DNS queries and while that socket is bound to a random port number it is never changed again. This goes against recommended practice since 2008, when Dan Kaminsky discovered how easy is to carry out DNS cache poisoning attacks. In order to prevent cache poisoning with spoofed DNS responses, it is necessary to maximise the uncertainty in the choice of a source port for a DNS query. Although unlikely in many setups, an external attacker could inject forged DNS responses into a Graylog's lookup table cache. In order to prevent this, it is at least recommendable to distribute the DNS queries through a pool of distinct sockets, each of them with a random source port and renew them periodically. This issue has been addressed in versions 5.0.9 and 5.1.3. Users are advised to upgrade. There are no known workarounds for this issue.", + "affected": [ + { + "ranges": [ + { + "type": "GIT", + "repo": "https://github.com/graylog2/graylog2-server", + "events": [ + { + "fixed": "466af814523cffae9fbc7e77bab7472988f03c3e" + }, + { + "fixed": "a101f4f12180fd3dfa7d3345188a099877a3c327" + } + ] + } + ] + } + ], + "references": [ + { + "type": "ADVISORY", + "url": "https://github.com/Graylog2/graylog2-server/security/advisories/GHSA-g96c-x7rh-99r3" + }, + { + "type": "EVIDENCE", + "url": "https://github.com/Graylog2/graylog2-server/security/advisories/GHSA-g96c-x7rh-99r3" + }, + { + "type": "FIX", + "url": "https://github.com/Graylog2/graylog2-server/commit/466af814523cffae9fbc7e77bab7472988f03c3e" + }, + { + "type": "FIX", + "url": "https://github.com/Graylog2/graylog2-server/commit/a101f4f12180fd3dfa7d3345188a099877a3c327" + } + ], + "aliases": [ + "GHSA-g96c-x7rh-99r3" + ], + "modified": "2023-09-06T20:02:28Z", + "published": "2023-08-31T18:15:09Z" +} From 750a53d2a2455e3d2618e69060379c4395bf271e Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 29 May 2024 02:31:53 +0000 Subject: [PATCH 02/32] Act on interim feedback - Generate the maps of checks and collections dynamically, to reduce future maintenance burden - Define an interface for Checks - Refactor existing code accordingly Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/checks.go | 125 +++++++++++++++++---- tools/osv-linter/internal/checks/ranges.go | 11 +- tools/osv-linter/internal/linter.go | 26 +++-- 3 files changed, 123 insertions(+), 39 deletions(-) diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index a6d45f21..c75937fe 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -1,43 +1,128 @@ package checks -import "github.com/tidwall/gjson" +import ( + "fmt" + "github.com/tidwall/gjson" +) + +// A CheckCode is a unique code for a check. type CheckCode string type CheckError struct { - Text string + Code CheckCode + Message string +} + +// Error returns the error message, including the code. +func (ce *CheckError) Error() string { + return fmt.Sprintf("%s: %s", ce.Code, ce.Message) +} + +// CodeString returns just the error code, as a string. +func (ce *CheckError) CodeString() string { + return string(ce.Code) +} + +// Checkers are for running a discrete checking function. +type Checker interface { + CodeString() string + Name() string + Description() string + Run(*gjson.Result) []CheckError } +// Check defines a single check. type Check struct { - Code CheckCode - Name string - Description string - Check func(*gjson.Result) []CheckError + code CheckCode + name string + description string + check func(*gjson.Result) []error +} + +// Run runs the check, returning any findings. +func (c *Check) Run(json *gjson.Result) (findings []CheckError) { + for _, finding := range c.check(json) { + findings = append(findings, CheckError{ + Code: c.code, + Message: finding.Error(), + }) + } + return findings +} + +// Name returns the name of the check. +func (c *Check) Name() string { + return c.name +} + +// Description returns the description of the check. +func (c *Check) Description() string { + return c.description } +// CodeString returns the short code for the check, as a string. +func (c *Check) CodeString() string { + return string(c.code) +} + +// CheckCollection is a named collection of checks. type CheckCollection struct { - Name string - Description string - Checks []*Check + name string + description string + checks []*Check +} + +// Name returns the name of the collection. +func (cc *CheckCollection) Name() string { + return cc.name +} + +// Description returns the description of the collection. +func (cc *CheckCollection) Description() string { + return cc.description +} + +// Checks returns the checks in the collection. +func (cc *CheckCollection) Checks() []*Check { + return cc.checks } var CheckIntroducedEventExists = &Check{ - Code: "R0001", - Name: "introduced-event-exists", - Description: "every range has an introduced event", - Check: RangeHasIntroducedEvent, + code: "R0001", + name: "introduced-event-exists", + description: "every range has an introduced event", + check: RangeHasIntroducedEvent, +} + +var checks = []*Check{ + CheckIntroducedEventExists, } -var AllChecks = map[string]*Check{ - "introduced-event-exists": CheckIntroducedEventExists, +// Checks returns all defined checks as a map, keyed by the check's code. +func Checks() (allchecks map[string]*Check) { + allchecks = make(map[string]*Check) + for _, check := range checks { + allchecks[check.CodeString()] = check + } + return allchecks } -var CheckCollections = map[string]CheckCollection{ - "osv.dev": { - Name: "osv.dev", - Description: "the checks OSV.dev considers necessary for a high quality record", - Checks: []*Check{ +var checkCollections = []CheckCollection{ + { + name: "osv.dev", + description: "the checks OSV.dev considers necessary for a high quality record", + checks: []*Check{ CheckIntroducedEventExists, }, }, } + +// CheckCollections returns a map of defined check collections, keyed by the collection's name. +func CheckCollections() (checkcollections map[string]CheckCollection) { + checkcollections = make(map[string]CheckCollection) + for _, checkcollection := range checkCollections { + checkcollections[checkcollection.Name()] = checkcollection + } + return checkcollections +} diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index 24d40265..db0a633d 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -6,20 +6,15 @@ import ( "github.com/tidwall/gjson" ) -func RangeHasIntroducedEvent(json *gjson.Result) []CheckError { +func RangeHasIntroducedEvent(json *gjson.Result) (findings []error) { result := json.Get(`affected.#.ranges.#.events`) - findings := []CheckError{} - result.ForEach(func(key, value gjson.Result) bool { if !value.Get("introduced").Exists() { - findings = append(findings, CheckError{Text: fmt.Sprintf("Error: Missing 'introduced' object in event at index %s", key)}) + findings = append(findings, fmt.Errorf("missing 'introduced' object in event at index %s", key)) } return true // Continue iteration. }) - if len(findings) != 0 { - return findings - } - return nil + return findings } diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index 7d45a9a1..85c95f79 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -16,16 +16,19 @@ import ( func LintCommand(cCtx *cli.Context) error { if cCtx.String("collection") == "list" { fmt.Printf("Available check collections:\n\n") - for _, collection := range checks.CheckCollections { - fmt.Printf("%s: %s\n", collection.Name, collection.Description) + for _, collection := range checks.CheckCollections() { + fmt.Printf("%s: %s\n", collection.Name(), collection.Description()) + for _, check := range collection.Checks() { + fmt.Printf("\t%s: (%s): %s\n", check.CodeString(), check.Name(), check.Description()) + } } return nil } if cCtx.String("check") == "list" { fmt.Printf("Available checks:\n\n") - for _, check := range checks.AllChecks { - fmt.Printf("%s: %s\n", check.Name, check.Description) + for _, check := range checks.Checks() { + fmt.Printf("%s: (%s): %s\n", check.CodeString(), check.Name(), check.Description()) } return nil } @@ -53,13 +56,13 @@ func LintCommand(cCtx *cli.Context) error { if cCtx.String("check") != "" { fmt.Printf("Running %q check on %q\n", cCtx.String("check"), fileToCheck) // Check the requested check exists. - if _, ok := checks.AllChecks[cCtx.String("check")]; !ok { + if _, ok := checks.Checks()[cCtx.String("check")]; !ok { return fmt.Errorf("%q is not a valid check", cCtx.String("check")) } // Run just the requested check. - check := checks.AllChecks[cCtx.String("check")] + check := checks.Checks()[cCtx.String("check")] // TODO: store in a per-file map so a per-file summary can be produced. - result := check.Check(&record) + result := check.Run(&record) if result != nil { log.Printf("%q: %q: %#v", fileToCheck, cCtx.String("check"), result) } @@ -69,15 +72,16 @@ func LintCommand(cCtx *cli.Context) error { if cCtx.String("collection") != "" { fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) // Check the requested check collection exists. - if _, ok := checks.CheckCollections[cCtx.String("collection")]; !ok { + if _, ok := checks.CheckCollections()[cCtx.String("collection")]; !ok { return fmt.Errorf("%q is not a valid check collection", cCtx.String("collection")) } // Run all checks in collection - for _, check := range checks.CheckCollections[cCtx.String("collection")].Checks { + collection := checks.CheckCollections()[cCtx.String("collection")] + for _, check := range collection.Checks() { // TODO: store in a per-file per-check map so a per-file summary can be produced. - result := check.Check(&record) + result := check.Run(&record) if result != nil { - log.Printf("%q: %q: %#v", fileToCheck, cCtx.String("check"), result) + log.Printf("%q: %q: %#v", fileToCheck, check.Name(), result) } } continue From 91f640a5d8c0ba4f1b7541a80e4353388930ee5f Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 29 May 2024 03:11:05 +0000 Subject: [PATCH 03/32] Adjust exported symbol naming to be more style compliant Add more docstrings Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/checks.go | 32 ++++++++++++++-------- tools/osv-linter/internal/checks/ranges.go | 1 + tools/osv-linter/internal/linter.go | 12 ++++---- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index c75937fe..7769bfa3 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -1,3 +1,13 @@ +// Package checks defines and implements all checks and collections of checks. +// +// To add additional checks: +// 1. define a new instance of `Check` +// 2. add it to the `checks` array +// 3. add it to the relevent collections defined in `checkCollections` +// +// To add additional collections of checks: +// 1. add to the `checkCollections` array. +// package checks import ( @@ -66,25 +76,25 @@ func (c *Check) CodeString() string { return string(c.code) } -// CheckCollection is a named collection of checks. -type CheckCollection struct { +// Collection is a named collection of checks. +type Collection struct { name string description string checks []*Check } // Name returns the name of the collection. -func (cc *CheckCollection) Name() string { +func (cc *Collection) Name() string { return cc.name } // Description returns the description of the collection. -func (cc *CheckCollection) Description() string { +func (cc *Collection) Description() string { return cc.description } // Checks returns the checks in the collection. -func (cc *CheckCollection) Checks() []*Check { +func (cc *Collection) Checks() []*Check { return cc.checks } @@ -99,8 +109,8 @@ var checks = []*Check{ CheckIntroducedEventExists, } -// Checks returns all defined checks as a map, keyed by the check's code. -func Checks() (allchecks map[string]*Check) { +// All returns all defined checks as a map, keyed by the check's code. +func All() (allchecks map[string]*Check) { allchecks = make(map[string]*Check) for _, check := range checks { allchecks[check.CodeString()] = check @@ -108,7 +118,7 @@ func Checks() (allchecks map[string]*Check) { return allchecks } -var checkCollections = []CheckCollection{ +var checkCollections = []Collection{ { name: "osv.dev", description: "the checks OSV.dev considers necessary for a high quality record", @@ -118,9 +128,9 @@ var checkCollections = []CheckCollection{ }, } -// CheckCollections returns a map of defined check collections, keyed by the collection's name. -func CheckCollections() (checkcollections map[string]CheckCollection) { - checkcollections = make(map[string]CheckCollection) +// Collections returns a map of defined check collections, keyed by the collection's name. +func Collections() (checkcollections map[string]Collection) { + checkcollections = make(map[string]Collection) for _, checkcollection := range checkCollections { checkcollections[checkcollection.Name()] = checkcollection } diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index db0a633d..5895cf71 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -6,6 +6,7 @@ import ( "github.com/tidwall/gjson" ) +// RangeHasIntroducedEvent checks for missing 'introduced' objects in events. func RangeHasIntroducedEvent(json *gjson.Result) (findings []error) { result := json.Get(`affected.#.ranges.#.events`) diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index 85c95f79..90705a7a 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -16,7 +16,7 @@ import ( func LintCommand(cCtx *cli.Context) error { if cCtx.String("collection") == "list" { fmt.Printf("Available check collections:\n\n") - for _, collection := range checks.CheckCollections() { + for _, collection := range checks.Collections() { fmt.Printf("%s: %s\n", collection.Name(), collection.Description()) for _, check := range collection.Checks() { fmt.Printf("\t%s: (%s): %s\n", check.CodeString(), check.Name(), check.Description()) @@ -27,7 +27,7 @@ func LintCommand(cCtx *cli.Context) error { if cCtx.String("check") == "list" { fmt.Printf("Available checks:\n\n") - for _, check := range checks.Checks() { + for _, check := range checks.All() { fmt.Printf("%s: (%s): %s\n", check.CodeString(), check.Name(), check.Description()) } return nil @@ -56,11 +56,11 @@ func LintCommand(cCtx *cli.Context) error { if cCtx.String("check") != "" { fmt.Printf("Running %q check on %q\n", cCtx.String("check"), fileToCheck) // Check the requested check exists. - if _, ok := checks.Checks()[cCtx.String("check")]; !ok { + if _, ok := checks.All()[cCtx.String("check")]; !ok { return fmt.Errorf("%q is not a valid check", cCtx.String("check")) } // Run just the requested check. - check := checks.Checks()[cCtx.String("check")] + check := checks.All()[cCtx.String("check")] // TODO: store in a per-file map so a per-file summary can be produced. result := check.Run(&record) if result != nil { @@ -72,11 +72,11 @@ func LintCommand(cCtx *cli.Context) error { if cCtx.String("collection") != "" { fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) // Check the requested check collection exists. - if _, ok := checks.CheckCollections()[cCtx.String("collection")]; !ok { + if _, ok := checks.Collections()[cCtx.String("collection")]; !ok { return fmt.Errorf("%q is not a valid check collection", cCtx.String("collection")) } // Run all checks in collection - collection := checks.CheckCollections()[cCtx.String("collection")] + collection := checks.Collections()[cCtx.String("collection")] for _, check := range collection.Checks() { // TODO: store in a per-file per-check map so a per-file summary can be produced. result := check.Run(&record) From fee2ae491fd51c8426bed9f14a413f6b66559c01 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 29 May 2024 07:08:17 +0000 Subject: [PATCH 04/32] Refactor to enable running on directories as well as individual files (actual directory walking support coming soon) Still functional: ``` $ go run cmd/osv/main.go record lint test_data/nointroduced-CVE-2023-41045.json Running "osv.dev" check collection on &["test_data/nointroduced-CVE-2023-41045.json"] Running "introduced-event-exists" check on "test_data/nointroduced-CVE-2023-41045.json" 2024/05/29 07:07:55 "test_data/nointroduced-CVE-2023-41045.json": "introduced-event-exists": []checks.CheckError{checks.CheckError{Code:"R0001", Message:"missing 'introduced' object in event at index 0"}} 2024/05/29 07:07:55 found errors exit status 1 ``` Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/checks.go | 2 + tools/osv-linter/internal/linter.go | 108 ++++++++++++++------- 2 files changed, 73 insertions(+), 37 deletions(-) diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index 7769bfa3..b8bbb83f 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -51,6 +51,8 @@ type Check struct { } // Run runs the check, returning any findings. +// The check has no awareness of the CheckCode, +// this merges that with the check's findings. func (c *Check) Run(json *gjson.Result) (findings []CheckError) { for _, finding := range c.check(json) { findings = append(findings, CheckError{ diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index 90705a7a..f3932172 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -13,7 +13,32 @@ import ( "github.com/ossf/osv-schema/linter/internal/checks" ) +type Content struct { + filename string + bytes []byte +} + +func lint(content *Content, checks []*checks.Check) (findings []checks.CheckError) { + // Parse file into JSON + if !gjson.ValidBytes(content.bytes) { + log.Printf("%q: invalid JSON", content.filename) + } + + record := gjson.ParseBytes(content.bytes) + + for _, check := range checks { + fmt.Printf("Running %q check on %q\n", check.Name(), content.filename) + checkFindings := check.Run(&record) + if checkFindings != nil { + log.Printf("%q: %q: %#v", content.filename, check.Name(), checkFindings) + } + findings = append(findings, checkFindings...) + } + return findings +} + func LintCommand(cCtx *cli.Context) error { + // List check collections. if cCtx.String("collection") == "list" { fmt.Printf("Available check collections:\n\n") for _, collection := range checks.Collections() { @@ -25,6 +50,7 @@ func LintCommand(cCtx *cli.Context) error { return nil } + // List all available checks. if cCtx.String("check") == "list" { fmt.Printf("Available checks:\n\n") for _, check := range checks.All() { @@ -33,59 +59,67 @@ func LintCommand(cCtx *cli.Context) error { return nil } + // Check for things to check. if cCtx.NArg() == 0 { return errors.New("nothing to check") } - for _, fileToCheck := range cCtx.Args().Slice() { + var checksToBeRun []*checks.Check - // Check file exists. - recordBytes, err := os.ReadFile(fileToCheck) - if err != nil { - log.Printf("%v, skipping", err) - continue + // Run the all the checks in a collection. + if cCtx.String("collection") != "" { + fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) + // Check the requested check collection exists. + if _, ok := checks.Collections()[cCtx.String("collection")]; !ok { + return fmt.Errorf("%q is not a valid check collection", cCtx.String("collection")) } + collection := checks.Collections()[cCtx.String("collection")] + checksToBeRun = collection.Checks() + } - // Parse file into JSON - if !gjson.ValidBytes(recordBytes) { - log.Printf("%q: invalid JSON", fileToCheck) + // Run just an individual check. + if cCtx.String("check") != "" { + // Check the requested check exists. + if _, ok := checks.All()[cCtx.String("check")]; !ok { + return fmt.Errorf("%q is not a valid check", cCtx.String("check")) } + checksToBeRun = append(checksToBeRun, checks.All()[cCtx.String("check")]) + } - record := gjson.ParseBytes(recordBytes) + perFileFindings := map[string][]checks.CheckError{} - if cCtx.String("check") != "" { - fmt.Printf("Running %q check on %q\n", cCtx.String("check"), fileToCheck) - // Check the requested check exists. - if _, ok := checks.All()[cCtx.String("check")]; !ok { - return fmt.Errorf("%q is not a valid check", cCtx.String("check")) - } - // Run just the requested check. - check := checks.All()[cCtx.String("check")] - // TODO: store in a per-file map so a per-file summary can be produced. - result := check.Run(&record) - if result != nil { - log.Printf("%q: %q: %#v", fileToCheck, cCtx.String("check"), result) - } + // Run the check(s) on the files. + for _, thingToCheck := range cCtx.Args().Slice() { + file, err := os.Open(thingToCheck) + if err != nil { + log.Printf("%v, skipping", err) continue } + defer file.Close() - if cCtx.String("collection") != "" { - fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) - // Check the requested check collection exists. - if _, ok := checks.Collections()[cCtx.String("collection")]; !ok { - return fmt.Errorf("%q is not a valid check collection", cCtx.String("collection")) + fileInfo, err := file.Stat() + if err != nil { + log.Printf("%v, skipping", err) + continue + } + + if fileInfo.IsDir() { + // Do the directory thing + } else { + // Do the file thing + recordBytes, err := os.ReadFile(thingToCheck) + if err != nil { + log.Printf("%v, skipping", err) + continue } - // Run all checks in collection - collection := checks.Collections()[cCtx.String("collection")] - for _, check := range collection.Checks() { - // TODO: store in a per-file per-check map so a per-file summary can be produced. - result := check.Run(&record) - if result != nil { - log.Printf("%q: %q: %#v", fileToCheck, check.Name(), result) - } + findings := lint(&Content{filename: thingToCheck, bytes: recordBytes}, checksToBeRun) + if findings != nil { + perFileFindings[thingToCheck] = findings } - continue } } + if len(perFileFindings) > 0 { + return errors.New("found errors") + } return nil } From 0ea104e60db7de52ad91eaf6f75dfd97b2e71e03 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Tue, 4 Jun 2024 04:21:31 +0000 Subject: [PATCH 05/32] Add test coverage and revise implementation I'm still getting the hang of GJSON's query syntax and how to operate on results from it. I'm at least now more confident about the behaviour of this check. Signed-off-by: Andrew Pollock --- tools/osv-linter/go.mod | 1 + tools/osv-linter/go.sum | 2 + tools/osv-linter/internal/checks/checks.go | 6 +-- tools/osv-linter/internal/checks/ranges.go | 18 +++---- .../osv-linter/internal/checks/ranges_test.go | 54 +++++++++++++++++++ .../osv-linter/test_data/CVE-2023-41045.json | 54 +++++++++++++++++++ 6 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 tools/osv-linter/internal/checks/ranges_test.go create mode 100644 tools/osv-linter/test_data/CVE-2023-41045.json diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index 14563a6b..ca3134bf 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -3,6 +3,7 @@ module github.com/ossf/osv-schema/linter go 1.21.10 require ( + github.com/google/go-cmp v0.6.0 github.com/tidwall/gjson v1.17.1 github.com/urfave/cli/v2 v2.27.2 ) diff --git a/tools/osv-linter/go.sum b/tools/osv-linter/go.sum index f2454fdd..bdf5c226 100644 --- a/tools/osv-linter/go.sum +++ b/tools/osv-linter/go.sum @@ -1,5 +1,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index b8bbb83f..fce3a21f 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -7,7 +7,6 @@ // // To add additional collections of checks: // 1. add to the `checkCollections` array. -// package checks import ( @@ -19,6 +18,7 @@ import ( // A CheckCode is a unique code for a check. type CheckCode string +// CheckError describes when a check fails. type CheckError struct { Code CheckCode Message string @@ -47,11 +47,11 @@ type Check struct { code CheckCode name string description string - check func(*gjson.Result) []error + check func(*gjson.Result) []CheckError } // Run runs the check, returning any findings. -// The check has no awareness of the CheckCode, +// The check has no awareness of the CheckCode, // this merges that with the check's findings. func (c *Check) Run(json *gjson.Result) (findings []CheckError) { for _, finding := range c.check(json) { diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index 5895cf71..2a9f8f71 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -1,21 +1,17 @@ package checks import ( - "fmt" - "github.com/tidwall/gjson" ) // RangeHasIntroducedEvent checks for missing 'introduced' objects in events. -func RangeHasIntroducedEvent(json *gjson.Result) (findings []error) { - result := json.Get(`affected.#.ranges.#.events`) +func RangeHasIntroducedEvent(json *gjson.Result) (findings []CheckError) { + result := json.Get(`affected.#(ranges.#(events.#(introduced)))`) - result.ForEach(func(key, value gjson.Result) bool { - if !value.Get("introduced").Exists() { - findings = append(findings, fmt.Errorf("missing 'introduced' object in event at index %s", key)) - } - return true // Continue iteration. - }) + if !result.Exists() { + findings = append(findings, CheckError{Message: "missing 'introduced' object in event"}) + return findings + } - return findings + return nil } diff --git a/tools/osv-linter/internal/checks/ranges_test.go b/tools/osv-linter/internal/checks/ranges_test.go new file mode 100644 index 00000000..61a35a07 --- /dev/null +++ b/tools/osv-linter/internal/checks/ranges_test.go @@ -0,0 +1,54 @@ +package checks + +import ( + "os" + "testing" + + "github.com/tidwall/gjson" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func loadTestData(filename string) *gjson.Result { + content, err := os.ReadFile(filename) + if err != nil { + panic(err) + } + record := gjson.ParseBytes(content) + return &record +} + +func TestRangeHasIntroducedEvent(t *testing.T) { + type args struct { + json *gjson.Result + } + tests := []struct { + name string + args args + wantFindings []CheckError + }{ + { + name: "A compliant file", + args: args{ + json: loadTestData("../../test_data/CVE-2023-41045.json"), + }, + wantFindings: nil, + }, + { + name: "A file without an introduced event", + args: args{ + json: loadTestData("../../test_data/nointroduced-CVE-2023-41045.json"), + }, + wantFindings: []CheckError{{Message: "missing 'introduced' object in event"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFindings := RangeHasIntroducedEvent(tt.args.json) + if diff := cmp.Diff(tt.wantFindings, gotFindings, cmpopts.EquateErrors()); diff != "" { + t.Errorf("RangeHasIntroducedEvent() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/tools/osv-linter/test_data/CVE-2023-41045.json b/tools/osv-linter/test_data/CVE-2023-41045.json new file mode 100644 index 00000000..88c4fe11 --- /dev/null +++ b/tools/osv-linter/test_data/CVE-2023-41045.json @@ -0,0 +1,54 @@ +{ + "id": "CVE-2023-41045", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:L/A:N" + } + ], + "details": "Graylog is a free and open log management platform. Graylog makes use of only one single source port for DNS queries. Graylog binds a single socket for outgoing DNS queries and while that socket is bound to a random port number it is never changed again. This goes against recommended practice since 2008, when Dan Kaminsky discovered how easy is to carry out DNS cache poisoning attacks. In order to prevent cache poisoning with spoofed DNS responses, it is necessary to maximise the uncertainty in the choice of a source port for a DNS query. Although unlikely in many setups, an external attacker could inject forged DNS responses into a Graylog's lookup table cache. In order to prevent this, it is at least recommendable to distribute the DNS queries through a pool of distinct sockets, each of them with a random source port and renew them periodically. This issue has been addressed in versions 5.0.9 and 5.1.3. Users are advised to upgrade. There are no known workarounds for this issue.", + "affected": [ + { + "ranges": [ + { + "type": "GIT", + "repo": "https://github.com/graylog2/graylog2-server", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "466af814523cffae9fbc7e77bab7472988f03c3e" + }, + { + "fixed": "a101f4f12180fd3dfa7d3345188a099877a3c327" + } + ] + } + ] + } + ], + "references": [ + { + "type": "ADVISORY", + "url": "https://github.com/Graylog2/graylog2-server/security/advisories/GHSA-g96c-x7rh-99r3" + }, + { + "type": "EVIDENCE", + "url": "https://github.com/Graylog2/graylog2-server/security/advisories/GHSA-g96c-x7rh-99r3" + }, + { + "type": "FIX", + "url": "https://github.com/Graylog2/graylog2-server/commit/466af814523cffae9fbc7e77bab7472988f03c3e" + }, + { + "type": "FIX", + "url": "https://github.com/Graylog2/graylog2-server/commit/a101f4f12180fd3dfa7d3345188a099877a3c327" + } + ], + "aliases": [ + "GHSA-g96c-x7rh-99r3" + ], + "modified": "2023-09-06T20:02:28Z", + "published": "2023-08-31T18:15:09Z" +} From ec85efd795bec7542cb81580469b824332e701ef Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Tue, 4 Jun 2024 05:13:04 +0000 Subject: [PATCH 06/32] Add directory support Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/linter.go | 39 ++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index f3932172..fbeb7f5e 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -3,8 +3,10 @@ package internal import ( "errors" "fmt" + "io/fs" "log" "os" + "path/filepath" "github.com/tidwall/gjson" @@ -88,7 +90,8 @@ func LintCommand(cCtx *cli.Context) error { perFileFindings := map[string][]checks.CheckError{} - // Run the check(s) on the files. + // Figure out what files to check. + var filesToCheck []string for _, thingToCheck := range cCtx.Args().Slice() { file, err := os.Open(thingToCheck) if err != nil { @@ -104,20 +107,38 @@ func LintCommand(cCtx *cli.Context) error { } if fileInfo.IsDir() { - // Do the directory thing - } else { - // Do the file thing - recordBytes, err := os.ReadFile(thingToCheck) + err := filepath.WalkDir(thingToCheck, func(f string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && filepath.Ext(d.Name()) == ".json" { + filesToCheck = append(filesToCheck, f) + } + return nil + }) if err != nil { log.Printf("%v, skipping", err) continue } - findings := lint(&Content{filename: thingToCheck, bytes: recordBytes}, checksToBeRun) - if findings != nil { - perFileFindings[thingToCheck] = findings - } + log.Printf("Found %d files in %q", len(filesToCheck), thingToCheck) + } else { + filesToCheck = append(filesToCheck, thingToCheck) } } + + // Run the check(s) on the files. + for _, fileToCheck := range filesToCheck { + recordBytes, err := os.ReadFile(fileToCheck) + if err != nil { + log.Printf("%v, skipping", err) + continue + } + findings := lint(&Content{filename: fileToCheck, bytes: recordBytes}, checksToBeRun) + if findings != nil { + perFileFindings[fileToCheck] = findings + } + } + if len(perFileFindings) > 0 { return errors.New("found errors") } From 82de09e710bf84da10022add5e97c857b3335ef5 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 19 Jun 2024 07:03:18 +0000 Subject: [PATCH 07/32] Tidy up after a pair-programming session with @agd Revert the interface, on the premise that there's only going to be one known implementation at this time. Rename the types, do away with the custom string and map. Move more of the definitional variable to the same place as the code. Simply how checks are inventoried by adding another "ALL" collection. Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/checks.go | 139 ++++++++------------- tools/osv-linter/internal/checks/ranges.go | 7 ++ tools/osv-linter/internal/linter.go | 37 +++--- 3 files changed, 78 insertions(+), 105 deletions(-) diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index fce3a21f..c777e4f3 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -15,12 +15,9 @@ import ( "github.com/tidwall/gjson" ) -// A CheckCode is a unique code for a check. -type CheckCode string - // CheckError describes when a check fails. type CheckError struct { - Code CheckCode + Code string Message string } @@ -29,112 +26,80 @@ func (ce *CheckError) Error() string { return fmt.Sprintf("%s: %s", ce.Code, ce.Message) } -// CodeString returns just the error code, as a string. -func (ce *CheckError) CodeString() string { - return string(ce.Code) -} - -// Checkers are for running a discrete checking function. -type Checker interface { - CodeString() string - Name() string - Description() string - Run(*gjson.Result) []CheckError +// CheckDef defines a single check. +type CheckDef struct { + Code string + Name string + Description string + Check Check } -// Check defines a single check. -type Check struct { - code CheckCode - name string - description string - check func(*gjson.Result) []CheckError -} +// Check defines how to run the check. +type Check func(*gjson.Result) []CheckError // Run runs the check, returning any findings. -// The check has no awareness of the CheckCode, +// The check has no awareness of the check's Code, // this merges that with the check's findings. -func (c *Check) Run(json *gjson.Result) (findings []CheckError) { - for _, finding := range c.check(json) { +func (c *CheckDef) Run(json *gjson.Result) (findings []CheckError) { + for _, finding := range c.Check(json) { findings = append(findings, CheckError{ - Code: c.code, + Code: c.Code, Message: finding.Error(), }) } return findings } -// Name returns the name of the check. -func (c *Check) Name() string { - return c.name -} - -// Description returns the description of the check. -func (c *Check) Description() string { - return c.description -} - -// CodeString returns the short code for the check, as a string. -func (c *Check) CodeString() string { - return string(c.code) -} - -// Collection is a named collection of checks. -type Collection struct { - name string - description string - checks []*Check -} - -// Name returns the name of the collection. -func (cc *Collection) Name() string { - return cc.name -} - -// Description returns the description of the collection. -func (cc *Collection) Description() string { - return cc.description +// CheckCollectionDef defines a named collection of checks. +type CheckCollectionDef struct { + Name string + Description string + Checks []*CheckDef } -// Checks returns the checks in the collection. -func (cc *Collection) Checks() []*Check { - return cc.checks -} - -var CheckIntroducedEventExists = &Check{ - code: "R0001", - name: "introduced-event-exists", - description: "every range has an introduced event", - check: RangeHasIntroducedEvent, -} - -var checks = []*Check{ - CheckIntroducedEventExists, +// FromCode returns the check with a specific code. +func FromCode(code string) *CheckDef { + for _, check := range CollectionFromName("ALL").Checks { + if check.Code == code { + return check + } + } + return nil } -// All returns all defined checks as a map, keyed by the check's code. -func All() (allchecks map[string]*Check) { - allchecks = make(map[string]*Check) - for _, check := range checks { - allchecks[check.CodeString()] = check +// FromName returns the check with a specific name. +func FromName(name string) *CheckDef { + for _, check := range CollectionFromName("ALL").Checks { + if check.Name == name { + return check + } } - return allchecks + return nil } -var checkCollections = []Collection{ +var Collections = []CheckCollectionDef{ + { + Name: "ALL", + Description: "all checks currently defined", + Checks: []*CheckDef{ + CheckRangeHasIntroducedEvent, + }, + }, { - name: "osv.dev", - description: "the checks OSV.dev considers necessary for a high quality record", - checks: []*Check{ - CheckIntroducedEventExists, + Name: "osv.dev", + Description: "the checks OSV.dev considers necessary for a high quality record", + Checks: []*CheckDef{ + CheckRangeHasIntroducedEvent, }, }, } -// Collections returns a map of defined check collections, keyed by the collection's name. -func Collections() (checkcollections map[string]Collection) { - checkcollections = make(map[string]Collection) - for _, checkcollection := range checkCollections { - checkcollections[checkcollection.Name()] = checkcollection +// CollectionFromName returns the CheckCollectionDef with the given name. +func CollectionFromName(name string) *CheckCollectionDef { + for _, checkcollection := range Collections { + if checkcollection.Name == name { + return &checkcollection + } } - return checkcollections + return nil } diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index 2a9f8f71..51ef0a75 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -4,6 +4,13 @@ import ( "github.com/tidwall/gjson" ) +var CheckRangeHasIntroducedEvent = &CheckDef{ + Code: "R0001", + Name: "introduced-event-exists", + Description: "every range has an introduced event", + Check: RangeHasIntroducedEvent, +} + // RangeHasIntroducedEvent checks for missing 'introduced' objects in events. func RangeHasIntroducedEvent(json *gjson.Result) (findings []CheckError) { result := json.Get(`affected.#(ranges.#(events.#(introduced)))`) diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index fbeb7f5e..17185ef1 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -20,7 +20,7 @@ type Content struct { bytes []byte } -func lint(content *Content, checks []*checks.Check) (findings []checks.CheckError) { +func lint(content *Content, checks []*checks.CheckDef) (findings []checks.CheckError) { // Parse file into JSON if !gjson.ValidBytes(content.bytes) { log.Printf("%q: invalid JSON", content.filename) @@ -29,10 +29,10 @@ func lint(content *Content, checks []*checks.Check) (findings []checks.CheckErro record := gjson.ParseBytes(content.bytes) for _, check := range checks { - fmt.Printf("Running %q check on %q\n", check.Name(), content.filename) + fmt.Printf("Running %q check on %q\n", check.Name, content.filename) checkFindings := check.Run(&record) if checkFindings != nil { - log.Printf("%q: %q: %#v", content.filename, check.Name(), checkFindings) + log.Printf("%q: %q: %#v", content.filename, check.Name, checkFindings) } findings = append(findings, checkFindings...) } @@ -43,10 +43,10 @@ func LintCommand(cCtx *cli.Context) error { // List check collections. if cCtx.String("collection") == "list" { fmt.Printf("Available check collections:\n\n") - for _, collection := range checks.Collections() { - fmt.Printf("%s: %s\n", collection.Name(), collection.Description()) - for _, check := range collection.Checks() { - fmt.Printf("\t%s: (%s): %s\n", check.CodeString(), check.Name(), check.Description()) + for _, collection := range checks.Collections { + fmt.Printf("%s: %s\n", collection.Name, collection.Description) + for _, check := range collection.Checks { + fmt.Printf("\t%s: (%s): %s\n", check.Code, check.Name, check.Description) } } return nil @@ -55,8 +55,8 @@ func LintCommand(cCtx *cli.Context) error { // List all available checks. if cCtx.String("check") == "list" { fmt.Printf("Available checks:\n\n") - for _, check := range checks.All() { - fmt.Printf("%s: (%s): %s\n", check.CodeString(), check.Name(), check.Description()) + for _, check := range checks.CollectionFromName("ALL").Checks { + fmt.Printf("%s: (%s): %s\n", check.Code, check.Name, check.Description) } return nil } @@ -66,26 +66,27 @@ func LintCommand(cCtx *cli.Context) error { return errors.New("nothing to check") } - var checksToBeRun []*checks.Check + var checksToBeRun []*checks.CheckDef - // Run the all the checks in a collection. + // Run all the checks in a collection. if cCtx.String("collection") != "" { fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) // Check the requested check collection exists. - if _, ok := checks.Collections()[cCtx.String("collection")]; !ok { + collection := checks.CollectionFromName(cCtx.String("collection")) + if collection == nil { return fmt.Errorf("%q is not a valid check collection", cCtx.String("collection")) } - collection := checks.Collections()[cCtx.String("collection")] - checksToBeRun = collection.Checks() + checksToBeRun = collection.Checks } // Run just an individual check. - if cCtx.String("check") != "" { + if code := cCtx.String("check"); code != "" { // Check the requested check exists. - if _, ok := checks.All()[cCtx.String("check")]; !ok { - return fmt.Errorf("%q is not a valid check", cCtx.String("check")) + check := checks.FromCode(code) + if check == nil { + return fmt.Errorf("%q is not a valid check", code) } - checksToBeRun = append(checksToBeRun, checks.All()[cCtx.String("check")]) + checksToBeRun = append(checksToBeRun, check) } perFileFindings := map[string][]checks.CheckError{} From b3da027733ddc0d27aadbd60556b91e4baa311b0 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Fri, 19 Jul 2024 05:23:05 +0000 Subject: [PATCH 08/32] Add check for ranges being distinct Use an historically incorrectly generated record that was flagged in https://github.com/google/osv.dev/issues/1984 ``` $ go run ./cmd/osv record lint test_data/CVE-2018-5407.json test_data/nondistinct-CVE-2018-5407.json Running "osv.dev" check collection on &["test_data/CVE-2018-5407.json" "test_data/nondistinct-CVE-2018-5407.json"] Running "introduced-event-exists" check on "test_data/CVE-2018-5407.json" Running "range-is-distinct" check on "test_data/CVE-2018-5407.json" Running "introduced-event-exists" check on "test_data/nondistinct-CVE-2018-5407.json" Running "range-is-distinct" check on "test_data/nondistinct-CVE-2018-5407.json" 2024/07/19 05:22:54 "test_data/nondistinct-CVE-2018-5407.json": "range-is-distinct": []checks.CheckError{checks.CheckError{Code:"R0002", Message:": overlapping event: \"e818b74be2170fbe957a07b0da4401c2b694b3b8\} 2024/07/19 05:22:54 found errors exit status 1 ``` Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/checks.go | 2 + tools/osv-linter/internal/checks/ranges.go | 71 ++ tools/osv-linter/test_data/CVE-2018-5407.json | 840 ++++++++++++++++++ .../test_data/nondistinct-CVE-2018-5407.json | 319 +++++++ 4 files changed, 1232 insertions(+) create mode 100644 tools/osv-linter/test_data/CVE-2018-5407.json create mode 100644 tools/osv-linter/test_data/nondistinct-CVE-2018-5407.json diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index c777e4f3..c431ab4f 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -83,6 +83,7 @@ var Collections = []CheckCollectionDef{ Description: "all checks currently defined", Checks: []*CheckDef{ CheckRangeHasIntroducedEvent, + CheckRangeIsDistinct, }, }, { @@ -90,6 +91,7 @@ var Collections = []CheckCollectionDef{ Description: "the checks OSV.dev considers necessary for a high quality record", Checks: []*CheckDef{ CheckRangeHasIntroducedEvent, + CheckRangeIsDistinct, }, }, } diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index 51ef0a75..7feed6ce 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -1,6 +1,9 @@ package checks import ( + "fmt" + "slices" + "github.com/tidwall/gjson" ) @@ -22,3 +25,71 @@ func RangeHasIntroducedEvent(json *gjson.Result) (findings []CheckError) { return nil } + +var CheckRangeIsDistinct = &CheckDef{ + Code: "R0002", + Name: "range-is-distinct", + Description: "range spans multiple versions/commits", + Check: RangeIsDistinct, +} + +type Range struct { + Beginning string + End string +} + +// RangeIsDistinct checks that the introduced and fixed (or last_affected) values differ. +// (on a per-repo basis for GIT ranges, and on a per-package basis otherwise) +func RangeIsDistinct(json *gjson.Result) (findings []CheckError) { + affectedEntries := json.Get(`affected`) + + // Examine each entry: + // for ones for packages, on a per-package basis + // for GIT ranges, on a per-repo basis + affectedEntries.ForEach(func(key, value gjson.Result) bool { + // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. + maybePackage := value.Get(`package`) + ranges := value.Get(`ranges`) + var pkg bool + if maybePackage.Exists() { + pkg = true + } + ranges.ForEach(func(key, value gjson.Result) bool { + rangeType := value.Get(`type`).String() + if !pkg && rangeType != "GIT" { + findings = append(findings, CheckError{Message: fmt.Sprintf("unexpected range type %q for %s", rangeType, value.String())}) + } + // Examine the events, collect all of the range starting values and range ending values (fixed and last_affected). + // There must be no overlap between these two sets of values. + events := value.Get(`events`) + var startEvents []string + var endEvents []string + events.ForEach(func(key, value gjson.Result) bool { + // Collect all the introduced values. + result := value.Get(`introduced`) + if result.Exists() { + startEvents = append(startEvents, result.String()) + } + // Collect all the fixed/last_affected values. + result = value.Get(`fixed`) + if result.Exists() { + endEvents = append(endEvents, result.String()) + } + result = value.Get(`last_affected`) + if result.Exists() { + endEvents = append(endEvents, result.String()) + } + return true // keep iterating (over events) + }) + // Check for overlap between collected start events and end events. + for _, endEvent := range endEvents { + if slices.Contains(startEvents, endEvent) { + findings = append(findings, CheckError{Message: fmt.Sprintf("overlapping event: %q", endEvent)}) + } + } + return true // keep iterating (over ranges) + }) + return true // keep iterating (over affected entries) + }) + return findings +} diff --git a/tools/osv-linter/test_data/CVE-2018-5407.json b/tools/osv-linter/test_data/CVE-2018-5407.json new file mode 100644 index 00000000..2c8d8a9d --- /dev/null +++ b/tools/osv-linter/test_data/CVE-2018-5407.json @@ -0,0 +1,840 @@ +{ + "id": "CVE-2018-5407", + "details": "Simultaneous Multi-threading (SMT) in processors can enable local users to exploit software vulnerable to timing attacks via a side-channel timing attack on 'port contention'.", + "modified": "2024-06-06T12:32:02.210372Z", + "published": "2018-11-15T21:29:00Z", + "related": [ + "DLA-1586-1", + "DSA-4348-1", + "DSA-4355-1", + "USN-3840-1" + ], + "references": [ + { + "type": "WEB", + "url": "http://www.securityfocus.com/bid/105897" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0483" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0651" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0652" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:2125" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3929" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3931" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3932" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3933" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3935" + }, + { + "type": "ADVISORY", + "url": "https://security.gentoo.org/glsa/201903-10" + }, + { + "type": "ADVISORY", + "url": "https://security.netapp.com/advisory/ntap-20181126-0001/" + }, + { + "type": "ADVISORY", + "url": "https://www.debian.org/security/2018/dsa-4348" + }, + { + "type": "ADVISORY", + "url": "https://www.debian.org/security/2018/dsa-4355" + }, + { + "type": "WEB", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujan2019-5072801.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujul2019-5072835.html" + }, + { + "type": "WEB", + "url": "https://lists.debian.org/debian-lts-announce/2018/11/msg00024.html" + }, + { + "type": "ARTICLE", + "url": "https://nodejs.org/en/blog/vulnerability/november-2018-security-releases/" + }, + { + "type": "WEB", + "url": "https://github.com/bbbrumley/portsmash" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpuapr2020.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujan2020.html" + }, + { + "type": "WEB", + "url": "https://eprint.iacr.org/2018/1060.pdf" + }, + { + "type": "WEB", + "url": "https://support.f5.com/csp/article/K49711130?utm_source=f5support&%3Butm_medium=RSS" + }, + { + "type": "WEB", + "url": "https://usn.ubuntu.com/3840-1/" + }, + { + "type": "WEB", + "url": "https://www.tenable.com/security/tns-2018-16" + }, + { + "type": "WEB", + "url": "https://www.tenable.com/security/tns-2018-17" + } + ], + "affected": [ + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.3", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ], + "versions": [ + "0.9.8i-r0", + "0.9.8j-r0", + "0.9.8k-r0", + "0.9.8k-r1", + "0.9.8k-r2", + "0.9.8k-r3", + "0.9.8k-r4", + "0.9.8k-r5", + "0.9.8k-r6", + "0.9.8k-r7", + "0.9.8l-r0", + "0.9.8l-r1", + "0.9.8m-r0", + "0.9.8n-r0", + "0.9.8n-r1", + "1.0.0-r0", + "1.0.0a-r0", + "1.0.0a-r1", + "1.0.0a-r2", + "1.0.0a-r3", + "1.0.0a-r4", + "1.0.0b-r0", + "1.0.0c-r0", + "1.0.0d-r0", + "1.0.0e-r0", + "1.0.0f-r0", + "1.0.0g-r0", + "1.0.0h-r0", + "1.0.1-r0", + "1.0.1a-r0", + "1.0.1b-r0", + "1.0.1c-r0", + "1.0.1c-r1", + "1.0.1c-r2", + "1.0.1c-r3", + "1.0.1d-r0", + "1.0.1d-r1", + "1.0.1e-r0", + "1.0.1e-r1", + "1.0.1e-r2", + "1.0.1e-r3", + "1.0.1e-r4", + "1.0.1e-r5", + "1.0.1e-r6", + "1.0.1e-r7", + "1.0.1f-r0", + "1.0.1g-r0", + "1.0.1g-r1", + "1.0.1g-r2", + "1.0.1g-r3", + "1.0.1h-r0", + "1.0.1i-r0", + "1.0.1i-r1", + "1.0.1i-r2", + "1.0.1i-r3", + "1.0.1j-r0", + "1.0.1k-r0", + "1.0.1l-r0", + "1.0.2-r0", + "1.0.2a-r0", + "1.0.2a-r1", + "1.0.2b-r0", + "1.0.2c-r0", + "1.0.2d-r0", + "1.0.2e-r0", + "1.0.2f-r0", + "1.0.2g-r0", + "1.0.2h-r0", + "1.0.2h-r1", + "1.0.2h-r2", + "1.0.2h-r3", + "1.0.2h-r4", + "1.0.2i-r0", + "1.0.2j-r0", + "1.0.2k-r0", + "1.0.2m-r0", + "1.0.2n-r0", + "1.0.2o-r0", + "1.0.2o-r1", + "1.0.2p-r0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.4", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ], + "versions": [ + "0.9.8i-r0", + "0.9.8j-r0", + "0.9.8k-r0", + "0.9.8k-r1", + "0.9.8k-r2", + "0.9.8k-r3", + "0.9.8k-r4", + "0.9.8k-r5", + "0.9.8k-r6", + "0.9.8k-r7", + "0.9.8l-r0", + "0.9.8l-r1", + "0.9.8m-r0", + "0.9.8n-r0", + "0.9.8n-r1", + "1.0.0-r0", + "1.0.0a-r0", + "1.0.0a-r1", + "1.0.0a-r2", + "1.0.0a-r3", + "1.0.0a-r4", + "1.0.0b-r0", + "1.0.0c-r0", + "1.0.0d-r0", + "1.0.0e-r0", + "1.0.0f-r0", + "1.0.0g-r0", + "1.0.0h-r0", + "1.0.1-r0", + "1.0.1a-r0", + "1.0.1b-r0", + "1.0.1c-r0", + "1.0.1c-r1", + "1.0.1c-r2", + "1.0.1c-r3", + "1.0.1d-r0", + "1.0.1d-r1", + "1.0.1e-r0", + "1.0.1e-r1", + "1.0.1e-r2", + "1.0.1e-r3", + "1.0.1e-r4", + "1.0.1e-r5", + "1.0.1e-r6", + "1.0.1e-r7", + "1.0.1f-r0", + "1.0.1g-r0", + "1.0.1g-r1", + "1.0.1g-r2", + "1.0.1g-r3", + "1.0.1h-r0", + "1.0.1i-r0", + "1.0.1i-r1", + "1.0.1i-r2", + "1.0.1i-r3", + "1.0.1j-r0", + "1.0.1k-r0", + "1.0.1l-r0", + "1.0.2-r0", + "1.0.2a-r0", + "1.0.2a-r1", + "1.0.2b-r0", + "1.0.2c-r0", + "1.0.2d-r0", + "1.0.2e-r0", + "1.0.2e-r1", + "1.0.2f-r0", + "1.0.2f-r1", + "1.0.2f-r2", + "1.0.2g-r0", + "1.0.2g-r1", + "1.0.2g-r2", + "1.0.2g-r3", + "1.0.2h-r0", + "1.0.2h-r1", + "1.0.2h-r2", + "1.0.2h-r3", + "1.0.2h-r4", + "1.0.2i-r0", + "1.0.2j-r0", + "1.0.2k-r0", + "1.0.2m-r0", + "1.0.2n-r0", + "1.0.2o-r0", + "1.0.2o-r1", + "1.0.2o-r2", + "1.0.2p-r0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.5", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ], + "versions": [ + "0.9.8i-r0", + "0.9.8j-r0", + "0.9.8k-r0", + "0.9.8k-r1", + "0.9.8k-r2", + "0.9.8k-r3", + "0.9.8k-r4", + "0.9.8k-r5", + "0.9.8k-r6", + "0.9.8k-r7", + "0.9.8l-r0", + "0.9.8l-r1", + "0.9.8m-r0", + "0.9.8n-r0", + "0.9.8n-r1", + "1.0.0-r0", + "1.0.0a-r0", + "1.0.0a-r1", + "1.0.0a-r2", + "1.0.0a-r3", + "1.0.0a-r4", + "1.0.0b-r0", + "1.0.0c-r0", + "1.0.0d-r0", + "1.0.0e-r0", + "1.0.0f-r0", + "1.0.0g-r0", + "1.0.0h-r0", + "1.0.1-r0", + "1.0.1a-r0", + "1.0.1b-r0", + "1.0.1c-r0", + "1.0.1c-r1", + "1.0.1c-r2", + "1.0.1c-r3", + "1.0.1d-r0", + "1.0.1d-r1", + "1.0.1e-r0", + "1.0.1e-r1", + "1.0.1e-r2", + "1.0.1e-r3", + "1.0.1e-r4", + "1.0.1e-r5", + "1.0.1e-r6", + "1.0.1e-r7", + "1.0.1f-r0", + "1.0.1g-r0", + "1.0.1g-r1", + "1.0.1g-r2", + "1.0.1g-r3", + "1.0.1h-r0", + "1.0.1i-r0", + "1.0.1i-r1", + "1.0.1i-r2", + "1.0.1i-r3", + "1.0.1j-r0", + "1.0.1k-r0", + "1.0.1l-r0", + "1.0.2-r0", + "1.0.2a-r0", + "1.0.2a-r1", + "1.0.2b-r0", + "1.0.2c-r0", + "1.0.2d-r0", + "1.0.2e-r0", + "1.0.2e-r1", + "1.0.2f-r0", + "1.0.2f-r1", + "1.0.2f-r2", + "1.0.2g-r0", + "1.0.2g-r1", + "1.0.2g-r2", + "1.0.2g-r3", + "1.0.2h-r0", + "1.0.2h-r1", + "1.0.2h-r2", + "1.0.2h-r3", + "1.0.2h-r4", + "1.0.2i-r0", + "1.0.2j-r0", + "1.0.2j-r1", + "1.0.2j-r2", + "1.0.2k-r0", + "1.0.2m-r0", + "1.0.2n-r0", + "1.0.2o-r0", + "1.0.2o-r1", + "1.0.2p-r0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.6", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ], + "versions": [ + "0.9.8i-r0", + "0.9.8j-r0", + "0.9.8k-r0", + "0.9.8k-r1", + "0.9.8k-r2", + "0.9.8k-r3", + "0.9.8k-r4", + "0.9.8k-r5", + "0.9.8k-r6", + "0.9.8k-r7", + "0.9.8l-r0", + "0.9.8l-r1", + "0.9.8m-r0", + "0.9.8n-r0", + "0.9.8n-r1", + "1.0.0-r0", + "1.0.0a-r0", + "1.0.0a-r1", + "1.0.0a-r2", + "1.0.0a-r3", + "1.0.0a-r4", + "1.0.0b-r0", + "1.0.0c-r0", + "1.0.0d-r0", + "1.0.0e-r0", + "1.0.0f-r0", + "1.0.0g-r0", + "1.0.0h-r0", + "1.0.1-r0", + "1.0.1a-r0", + "1.0.1b-r0", + "1.0.1c-r0", + "1.0.1c-r1", + "1.0.1c-r2", + "1.0.1c-r3", + "1.0.1d-r0", + "1.0.1d-r1", + "1.0.1e-r0", + "1.0.1e-r1", + "1.0.1e-r2", + "1.0.1e-r3", + "1.0.1e-r4", + "1.0.1e-r5", + "1.0.1e-r6", + "1.0.1e-r7", + "1.0.1f-r0", + "1.0.1g-r0", + "1.0.1g-r1", + "1.0.1g-r2", + "1.0.1g-r3", + "1.0.1h-r0", + "1.0.1i-r0", + "1.0.1i-r1", + "1.0.1i-r2", + "1.0.1i-r3", + "1.0.1j-r0", + "1.0.1k-r0", + "1.0.1l-r0", + "1.0.2-r0", + "1.0.2a-r0", + "1.0.2a-r1", + "1.0.2b-r0", + "1.0.2c-r0", + "1.0.2d-r0", + "1.0.2e-r0", + "1.0.2e-r1", + "1.0.2f-r0", + "1.0.2f-r1", + "1.0.2f-r2", + "1.0.2g-r0", + "1.0.2g-r1", + "1.0.2g-r2", + "1.0.2g-r3", + "1.0.2h-r0", + "1.0.2h-r1", + "1.0.2h-r2", + "1.0.2h-r3", + "1.0.2h-r4", + "1.0.2i-r0", + "1.0.2j-r0", + "1.0.2j-r1", + "1.0.2j-r2", + "1.0.2k-r0", + "1.0.2l-r0", + "1.0.2m-r0", + "1.0.2n-r0", + "1.0.2o-r0", + "1.0.2o-r1", + "1.0.2p-r0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.7", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ], + "versions": [ + "0.9.8i-r0", + "0.9.8j-r0", + "0.9.8k-r0", + "0.9.8k-r1", + "0.9.8k-r2", + "0.9.8k-r3", + "0.9.8k-r4", + "0.9.8k-r5", + "0.9.8k-r6", + "0.9.8k-r7", + "0.9.8l-r0", + "0.9.8l-r1", + "0.9.8m-r0", + "0.9.8n-r0", + "0.9.8n-r1", + "1.0.0-r0", + "1.0.0a-r0", + "1.0.0a-r1", + "1.0.0a-r2", + "1.0.0a-r3", + "1.0.0a-r4", + "1.0.0b-r0", + "1.0.0c-r0", + "1.0.0d-r0", + "1.0.0e-r0", + "1.0.0f-r0", + "1.0.0g-r0", + "1.0.0h-r0", + "1.0.1-r0", + "1.0.1a-r0", + "1.0.1b-r0", + "1.0.1c-r0", + "1.0.1c-r1", + "1.0.1c-r2", + "1.0.1c-r3", + "1.0.1d-r0", + "1.0.1d-r1", + "1.0.1e-r0", + "1.0.1e-r1", + "1.0.1e-r2", + "1.0.1e-r3", + "1.0.1e-r4", + "1.0.1e-r5", + "1.0.1e-r6", + "1.0.1e-r7", + "1.0.1f-r0", + "1.0.1g-r0", + "1.0.1g-r1", + "1.0.1g-r2", + "1.0.1g-r3", + "1.0.1h-r0", + "1.0.1i-r0", + "1.0.1i-r1", + "1.0.1i-r2", + "1.0.1i-r3", + "1.0.1j-r0", + "1.0.1k-r0", + "1.0.1l-r0", + "1.0.2-r0", + "1.0.2a-r0", + "1.0.2a-r1", + "1.0.2b-r0", + "1.0.2c-r0", + "1.0.2d-r0", + "1.0.2e-r0", + "1.0.2e-r1", + "1.0.2f-r0", + "1.0.2f-r1", + "1.0.2f-r2", + "1.0.2g-r0", + "1.0.2g-r1", + "1.0.2g-r2", + "1.0.2g-r3", + "1.0.2h-r0", + "1.0.2h-r1", + "1.0.2h-r2", + "1.0.2h-r3", + "1.0.2h-r4", + "1.0.2i-r0", + "1.0.2j-r0", + "1.0.2j-r1", + "1.0.2j-r2", + "1.0.2k-r0", + "1.0.2l-r0", + "1.0.2m-r0", + "1.0.2n-r0", + "1.0.2o-r0", + "1.0.2o-r1", + "1.0.2p-r0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.8", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ], + "versions": [ + "0.9.8i-r0", + "0.9.8j-r0", + "0.9.8k-r0", + "0.9.8k-r1", + "0.9.8k-r2", + "0.9.8k-r3", + "0.9.8k-r4", + "0.9.8k-r5", + "0.9.8k-r6", + "0.9.8k-r7", + "0.9.8l-r0", + "0.9.8l-r1", + "0.9.8m-r0", + "0.9.8n-r0", + "0.9.8n-r1", + "1.0.0-r0", + "1.0.0a-r0", + "1.0.0a-r1", + "1.0.0a-r2", + "1.0.0a-r3", + "1.0.0a-r4", + "1.0.0b-r0", + "1.0.0c-r0", + "1.0.0d-r0", + "1.0.0e-r0", + "1.0.0f-r0", + "1.0.0g-r0", + "1.0.0h-r0", + "1.0.1-r0", + "1.0.1a-r0", + "1.0.1b-r0", + "1.0.1c-r0", + "1.0.1c-r1", + "1.0.1c-r2", + "1.0.1c-r3", + "1.0.1d-r0", + "1.0.1d-r1", + "1.0.1e-r0", + "1.0.1e-r1", + "1.0.1e-r2", + "1.0.1e-r3", + "1.0.1e-r4", + "1.0.1e-r5", + "1.0.1e-r6", + "1.0.1e-r7", + "1.0.1f-r0", + "1.0.1g-r0", + "1.0.1g-r1", + "1.0.1g-r2", + "1.0.1g-r3", + "1.0.1h-r0", + "1.0.1i-r0", + "1.0.1i-r1", + "1.0.1i-r2", + "1.0.1i-r3", + "1.0.1j-r0", + "1.0.1k-r0", + "1.0.1l-r0", + "1.0.2-r0", + "1.0.2a-r0", + "1.0.2a-r1", + "1.0.2b-r0", + "1.0.2c-r0", + "1.0.2d-r0", + "1.0.2e-r0", + "1.0.2e-r1", + "1.0.2f-r0", + "1.0.2f-r1", + "1.0.2f-r2", + "1.0.2g-r0", + "1.0.2g-r1", + "1.0.2g-r2", + "1.0.2g-r3", + "1.0.2h-r0", + "1.0.2h-r1", + "1.0.2h-r2", + "1.0.2h-r3", + "1.0.2h-r4", + "1.0.2i-r0", + "1.0.2j-r0", + "1.0.2j-r1", + "1.0.2j-r2", + "1.0.2k-r0", + "1.0.2l-r0", + "1.0.2m-r0", + "1.0.2n-r0", + "1.0.2o-r0", + "1.0.2o-r1", + "1.0.2o-r2", + "1.0.2p-r0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + }, + { + "ranges": [ + { + "type": "GIT", + "repo": "https://github.com/nodejs/node", + "events": [ + { + "introduced": "cf41627411886000429bde058a6594fb7f6d6d47" + }, + { + "fixed": "03b825811ed4af0addcdf6e75bacb3dc1c4c5940" + } + ] + } + ], + "versions": [ + "v10.0.0", + "v10.1.0", + "v10.2.0", + "v10.2.1", + "v10.3.0", + "v10.4.0", + "v10.4.1", + "v10.5.0", + "v10.6.0", + "v10.7.0", + "v10.8.0" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" + } + } + ], + "schema_version": "1.6.0", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:H/I:N/A:N" + } + ] +} diff --git a/tools/osv-linter/test_data/nondistinct-CVE-2018-5407.json b/tools/osv-linter/test_data/nondistinct-CVE-2018-5407.json new file mode 100644 index 00000000..e9ad70aa --- /dev/null +++ b/tools/osv-linter/test_data/nondistinct-CVE-2018-5407.json @@ -0,0 +1,319 @@ +{ + "id": "CVE-2018-5407", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:H/I:N/A:N" + } + ], + "details": "Simultaneous Multi-threading (SMT) in processors can enable local users to exploit software vulnerable to timing attacks via a side-channel timing attack on 'port contention'.", + "affected": [ + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.3", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ] + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.4", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ] + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.5", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ] + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.6", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ] + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.7", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ] + }, + { + "package": { + "name": "openssl", + "ecosystem": "Alpine:v3.8", + "purl": "pkg:apk/alpine/openssl?arch=source" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2q-r0" + } + ] + } + ] + }, + { + "ranges": [ + { + "type": "GIT", + "repo": "https://github.com/nodejs/node", + "events": [ + { + "introduced": "cf41627411886000429bde058a6594fb7f6d6d47" + }, + { + "fixed": "03b825811ed4af0addcdf6e75bacb3dc1c4c5940" + } + ] + }, + { + "type": "GIT", + "repo": "https://github.com/openssl/openssl", + "events": [ + { + "introduced": "e818b74be2170fbe957a07b0da4401c2b694b3b8" + }, + { + "fixed": "e818b74be2170fbe957a07b0da4401c2b694b3b8" + }, + { + "introduced": "7ea5bd2b52d0e81eaef3d109b3b12545306f201c" + } + ] + } + ] + } + ], + "references": [ + { + "type": "ADVISORY", + "url": "http://www.securityfocus.com/bid/105897" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0483" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0651" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0652" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:2125" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3929" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3931" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3932" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3933" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3935" + }, + { + "type": "ADVISORY", + "url": "https://security.gentoo.org/glsa/201903-10" + }, + { + "type": "ADVISORY", + "url": "https://security.netapp.com/advisory/ntap-20181126-0001/" + }, + { + "type": "ADVISORY", + "url": "https://www.debian.org/security/2018/dsa-4348" + }, + { + "type": "ADVISORY", + "url": "https://www.debian.org/security/2018/dsa-4355" + }, + { + "type": "ADVISORY", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "ADVISORY", + "url": "https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" + }, + { + "type": "ADVISORY", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujan2019-5072801.html" + }, + { + "type": "ADVISORY", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujul2019-5072835.html" + }, + { + "type": "ARTICLE", + "url": "https://lists.debian.org/debian-lts-announce/2018/11/msg00024.html" + }, + { + "type": "ARTICLE", + "url": "https://nodejs.org/en/blog/vulnerability/november-2018-security-releases/" + }, + { + "type": "EVIDENCE", + "url": "https://github.com/bbbrumley/portsmash" + }, + { + "type": "EVIDENCE", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/security-alerts/cpuapr2020.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/security-alerts/cpujan2020.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujan2019-5072801.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujul2019-5072835.html" + }, + { + "type": "WEB", + "url": "http://www.securityfocus.com/bid/105897" + }, + { + "type": "WEB", + "url": "https://eprint.iacr.org/2018/1060.pdf" + }, + { + "type": "WEB", + "url": "https://github.com/bbbrumley/portsmash" + }, + { + "type": "WEB", + "url": "https://lists.debian.org/debian-lts-announce/2018/11/msg00024.html" + }, + { + "type": "WEB", + "url": "https://support.f5.com/csp/article/K49711130?utm_source=f5support\u0026amp%3Butm_medium=RSS" + }, + { + "type": "WEB", + "url": "https://usn.ubuntu.com/3840-1/" + }, + { + "type": "WEB", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpuapr2020.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujan2020.html" + }, + { + "type": "WEB", + "url": "https://www.tenable.com/security/tns-2018-16" + }, + { + "type": "WEB", + "url": "https://www.tenable.com/security/tns-2018-17" + } + ], + "modified": "2024-05-29T23:34:56Z", + "published": "2018-11-15T21:29:00Z" +} From c663b59868bb5a740e48af317974dc35065c1f07 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Fri, 19 Jul 2024 05:29:22 +0000 Subject: [PATCH 09/32] Use the pre-import version of CVE-2018-5407 For more conciseness, and because the linter is expected to be run on records before they're processed by OSV.dev, not after. Signed-off-by: Andrew Pollock --- tools/osv-linter/test_data/CVE-2018-5407.json | 881 ++++-------------- 1 file changed, 180 insertions(+), 701 deletions(-) diff --git a/tools/osv-linter/test_data/CVE-2018-5407.json b/tools/osv-linter/test_data/CVE-2018-5407.json index 2c8d8a9d..e9ad70aa 100644 --- a/tools/osv-linter/test_data/CVE-2018-5407.json +++ b/tools/osv-linter/test_data/CVE-2018-5407.json @@ -1,128 +1,12 @@ { "id": "CVE-2018-5407", - "details": "Simultaneous Multi-threading (SMT) in processors can enable local users to exploit software vulnerable to timing attacks via a side-channel timing attack on 'port contention'.", - "modified": "2024-06-06T12:32:02.210372Z", - "published": "2018-11-15T21:29:00Z", - "related": [ - "DLA-1586-1", - "DSA-4348-1", - "DSA-4355-1", - "USN-3840-1" - ], - "references": [ - { - "type": "WEB", - "url": "http://www.securityfocus.com/bid/105897" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:0483" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:0651" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:0652" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:2125" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:3929" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:3931" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:3932" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:3933" - }, - { - "type": "ADVISORY", - "url": "https://access.redhat.com/errata/RHSA-2019:3935" - }, - { - "type": "ADVISORY", - "url": "https://security.gentoo.org/glsa/201903-10" - }, - { - "type": "ADVISORY", - "url": "https://security.netapp.com/advisory/ntap-20181126-0001/" - }, - { - "type": "ADVISORY", - "url": "https://www.debian.org/security/2018/dsa-4348" - }, - { - "type": "ADVISORY", - "url": "https://www.debian.org/security/2018/dsa-4355" - }, - { - "type": "WEB", - "url": "https://www.exploit-db.com/exploits/45785/" - }, - { - "type": "FIX", - "url": "https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" - }, - { - "type": "FIX", - "url": "https://www.oracle.com/technetwork/security-advisory/cpujan2019-5072801.html" - }, - { - "type": "FIX", - "url": "https://www.oracle.com/technetwork/security-advisory/cpujul2019-5072835.html" - }, - { - "type": "WEB", - "url": "https://lists.debian.org/debian-lts-announce/2018/11/msg00024.html" - }, - { - "type": "ARTICLE", - "url": "https://nodejs.org/en/blog/vulnerability/november-2018-security-releases/" - }, - { - "type": "WEB", - "url": "https://github.com/bbbrumley/portsmash" - }, - { - "type": "WEB", - "url": "https://www.oracle.com/security-alerts/cpuapr2020.html" - }, - { - "type": "WEB", - "url": "https://www.oracle.com/security-alerts/cpujan2020.html" - }, - { - "type": "WEB", - "url": "https://eprint.iacr.org/2018/1060.pdf" - }, - { - "type": "WEB", - "url": "https://support.f5.com/csp/article/K49711130?utm_source=f5support&%3Butm_medium=RSS" - }, - { - "type": "WEB", - "url": "https://usn.ubuntu.com/3840-1/" - }, - { - "type": "WEB", - "url": "https://www.tenable.com/security/tns-2018-16" - }, + "severity": [ { - "type": "WEB", - "url": "https://www.tenable.com/security/tns-2018-17" + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:H/I:N/A:N" } ], + "details": "Simultaneous Multi-threading (SMT) in processors can enable local users to exploit software vulnerable to timing attacks via a side-channel timing attack on 'port contention'.", "affected": [ { "package": { @@ -142,92 +26,7 @@ } ] } - ], - "versions": [ - "0.9.8i-r0", - "0.9.8j-r0", - "0.9.8k-r0", - "0.9.8k-r1", - "0.9.8k-r2", - "0.9.8k-r3", - "0.9.8k-r4", - "0.9.8k-r5", - "0.9.8k-r6", - "0.9.8k-r7", - "0.9.8l-r0", - "0.9.8l-r1", - "0.9.8m-r0", - "0.9.8n-r0", - "0.9.8n-r1", - "1.0.0-r0", - "1.0.0a-r0", - "1.0.0a-r1", - "1.0.0a-r2", - "1.0.0a-r3", - "1.0.0a-r4", - "1.0.0b-r0", - "1.0.0c-r0", - "1.0.0d-r0", - "1.0.0e-r0", - "1.0.0f-r0", - "1.0.0g-r0", - "1.0.0h-r0", - "1.0.1-r0", - "1.0.1a-r0", - "1.0.1b-r0", - "1.0.1c-r0", - "1.0.1c-r1", - "1.0.1c-r2", - "1.0.1c-r3", - "1.0.1d-r0", - "1.0.1d-r1", - "1.0.1e-r0", - "1.0.1e-r1", - "1.0.1e-r2", - "1.0.1e-r3", - "1.0.1e-r4", - "1.0.1e-r5", - "1.0.1e-r6", - "1.0.1e-r7", - "1.0.1f-r0", - "1.0.1g-r0", - "1.0.1g-r1", - "1.0.1g-r2", - "1.0.1g-r3", - "1.0.1h-r0", - "1.0.1i-r0", - "1.0.1i-r1", - "1.0.1i-r2", - "1.0.1i-r3", - "1.0.1j-r0", - "1.0.1k-r0", - "1.0.1l-r0", - "1.0.2-r0", - "1.0.2a-r0", - "1.0.2a-r1", - "1.0.2b-r0", - "1.0.2c-r0", - "1.0.2d-r0", - "1.0.2e-r0", - "1.0.2f-r0", - "1.0.2g-r0", - "1.0.2h-r0", - "1.0.2h-r1", - "1.0.2h-r2", - "1.0.2h-r3", - "1.0.2h-r4", - "1.0.2i-r0", - "1.0.2j-r0", - "1.0.2k-r0", - "1.0.2m-r0", - "1.0.2n-r0", - "1.0.2o-r0", - "1.0.2o-r1", - "1.0.2p-r0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] }, { "package": { @@ -247,99 +46,7 @@ } ] } - ], - "versions": [ - "0.9.8i-r0", - "0.9.8j-r0", - "0.9.8k-r0", - "0.9.8k-r1", - "0.9.8k-r2", - "0.9.8k-r3", - "0.9.8k-r4", - "0.9.8k-r5", - "0.9.8k-r6", - "0.9.8k-r7", - "0.9.8l-r0", - "0.9.8l-r1", - "0.9.8m-r0", - "0.9.8n-r0", - "0.9.8n-r1", - "1.0.0-r0", - "1.0.0a-r0", - "1.0.0a-r1", - "1.0.0a-r2", - "1.0.0a-r3", - "1.0.0a-r4", - "1.0.0b-r0", - "1.0.0c-r0", - "1.0.0d-r0", - "1.0.0e-r0", - "1.0.0f-r0", - "1.0.0g-r0", - "1.0.0h-r0", - "1.0.1-r0", - "1.0.1a-r0", - "1.0.1b-r0", - "1.0.1c-r0", - "1.0.1c-r1", - "1.0.1c-r2", - "1.0.1c-r3", - "1.0.1d-r0", - "1.0.1d-r1", - "1.0.1e-r0", - "1.0.1e-r1", - "1.0.1e-r2", - "1.0.1e-r3", - "1.0.1e-r4", - "1.0.1e-r5", - "1.0.1e-r6", - "1.0.1e-r7", - "1.0.1f-r0", - "1.0.1g-r0", - "1.0.1g-r1", - "1.0.1g-r2", - "1.0.1g-r3", - "1.0.1h-r0", - "1.0.1i-r0", - "1.0.1i-r1", - "1.0.1i-r2", - "1.0.1i-r3", - "1.0.1j-r0", - "1.0.1k-r0", - "1.0.1l-r0", - "1.0.2-r0", - "1.0.2a-r0", - "1.0.2a-r1", - "1.0.2b-r0", - "1.0.2c-r0", - "1.0.2d-r0", - "1.0.2e-r0", - "1.0.2e-r1", - "1.0.2f-r0", - "1.0.2f-r1", - "1.0.2f-r2", - "1.0.2g-r0", - "1.0.2g-r1", - "1.0.2g-r2", - "1.0.2g-r3", - "1.0.2h-r0", - "1.0.2h-r1", - "1.0.2h-r2", - "1.0.2h-r3", - "1.0.2h-r4", - "1.0.2i-r0", - "1.0.2j-r0", - "1.0.2k-r0", - "1.0.2m-r0", - "1.0.2n-r0", - "1.0.2o-r0", - "1.0.2o-r1", - "1.0.2o-r2", - "1.0.2p-r0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] }, { "package": { @@ -359,100 +66,7 @@ } ] } - ], - "versions": [ - "0.9.8i-r0", - "0.9.8j-r0", - "0.9.8k-r0", - "0.9.8k-r1", - "0.9.8k-r2", - "0.9.8k-r3", - "0.9.8k-r4", - "0.9.8k-r5", - "0.9.8k-r6", - "0.9.8k-r7", - "0.9.8l-r0", - "0.9.8l-r1", - "0.9.8m-r0", - "0.9.8n-r0", - "0.9.8n-r1", - "1.0.0-r0", - "1.0.0a-r0", - "1.0.0a-r1", - "1.0.0a-r2", - "1.0.0a-r3", - "1.0.0a-r4", - "1.0.0b-r0", - "1.0.0c-r0", - "1.0.0d-r0", - "1.0.0e-r0", - "1.0.0f-r0", - "1.0.0g-r0", - "1.0.0h-r0", - "1.0.1-r0", - "1.0.1a-r0", - "1.0.1b-r0", - "1.0.1c-r0", - "1.0.1c-r1", - "1.0.1c-r2", - "1.0.1c-r3", - "1.0.1d-r0", - "1.0.1d-r1", - "1.0.1e-r0", - "1.0.1e-r1", - "1.0.1e-r2", - "1.0.1e-r3", - "1.0.1e-r4", - "1.0.1e-r5", - "1.0.1e-r6", - "1.0.1e-r7", - "1.0.1f-r0", - "1.0.1g-r0", - "1.0.1g-r1", - "1.0.1g-r2", - "1.0.1g-r3", - "1.0.1h-r0", - "1.0.1i-r0", - "1.0.1i-r1", - "1.0.1i-r2", - "1.0.1i-r3", - "1.0.1j-r0", - "1.0.1k-r0", - "1.0.1l-r0", - "1.0.2-r0", - "1.0.2a-r0", - "1.0.2a-r1", - "1.0.2b-r0", - "1.0.2c-r0", - "1.0.2d-r0", - "1.0.2e-r0", - "1.0.2e-r1", - "1.0.2f-r0", - "1.0.2f-r1", - "1.0.2f-r2", - "1.0.2g-r0", - "1.0.2g-r1", - "1.0.2g-r2", - "1.0.2g-r3", - "1.0.2h-r0", - "1.0.2h-r1", - "1.0.2h-r2", - "1.0.2h-r3", - "1.0.2h-r4", - "1.0.2i-r0", - "1.0.2j-r0", - "1.0.2j-r1", - "1.0.2j-r2", - "1.0.2k-r0", - "1.0.2m-r0", - "1.0.2n-r0", - "1.0.2o-r0", - "1.0.2o-r1", - "1.0.2p-r0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] }, { "package": { @@ -472,101 +86,7 @@ } ] } - ], - "versions": [ - "0.9.8i-r0", - "0.9.8j-r0", - "0.9.8k-r0", - "0.9.8k-r1", - "0.9.8k-r2", - "0.9.8k-r3", - "0.9.8k-r4", - "0.9.8k-r5", - "0.9.8k-r6", - "0.9.8k-r7", - "0.9.8l-r0", - "0.9.8l-r1", - "0.9.8m-r0", - "0.9.8n-r0", - "0.9.8n-r1", - "1.0.0-r0", - "1.0.0a-r0", - "1.0.0a-r1", - "1.0.0a-r2", - "1.0.0a-r3", - "1.0.0a-r4", - "1.0.0b-r0", - "1.0.0c-r0", - "1.0.0d-r0", - "1.0.0e-r0", - "1.0.0f-r0", - "1.0.0g-r0", - "1.0.0h-r0", - "1.0.1-r0", - "1.0.1a-r0", - "1.0.1b-r0", - "1.0.1c-r0", - "1.0.1c-r1", - "1.0.1c-r2", - "1.0.1c-r3", - "1.0.1d-r0", - "1.0.1d-r1", - "1.0.1e-r0", - "1.0.1e-r1", - "1.0.1e-r2", - "1.0.1e-r3", - "1.0.1e-r4", - "1.0.1e-r5", - "1.0.1e-r6", - "1.0.1e-r7", - "1.0.1f-r0", - "1.0.1g-r0", - "1.0.1g-r1", - "1.0.1g-r2", - "1.0.1g-r3", - "1.0.1h-r0", - "1.0.1i-r0", - "1.0.1i-r1", - "1.0.1i-r2", - "1.0.1i-r3", - "1.0.1j-r0", - "1.0.1k-r0", - "1.0.1l-r0", - "1.0.2-r0", - "1.0.2a-r0", - "1.0.2a-r1", - "1.0.2b-r0", - "1.0.2c-r0", - "1.0.2d-r0", - "1.0.2e-r0", - "1.0.2e-r1", - "1.0.2f-r0", - "1.0.2f-r1", - "1.0.2f-r2", - "1.0.2g-r0", - "1.0.2g-r1", - "1.0.2g-r2", - "1.0.2g-r3", - "1.0.2h-r0", - "1.0.2h-r1", - "1.0.2h-r2", - "1.0.2h-r3", - "1.0.2h-r4", - "1.0.2i-r0", - "1.0.2j-r0", - "1.0.2j-r1", - "1.0.2j-r2", - "1.0.2k-r0", - "1.0.2l-r0", - "1.0.2m-r0", - "1.0.2n-r0", - "1.0.2o-r0", - "1.0.2o-r1", - "1.0.2p-r0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] }, { "package": { @@ -586,101 +106,7 @@ } ] } - ], - "versions": [ - "0.9.8i-r0", - "0.9.8j-r0", - "0.9.8k-r0", - "0.9.8k-r1", - "0.9.8k-r2", - "0.9.8k-r3", - "0.9.8k-r4", - "0.9.8k-r5", - "0.9.8k-r6", - "0.9.8k-r7", - "0.9.8l-r0", - "0.9.8l-r1", - "0.9.8m-r0", - "0.9.8n-r0", - "0.9.8n-r1", - "1.0.0-r0", - "1.0.0a-r0", - "1.0.0a-r1", - "1.0.0a-r2", - "1.0.0a-r3", - "1.0.0a-r4", - "1.0.0b-r0", - "1.0.0c-r0", - "1.0.0d-r0", - "1.0.0e-r0", - "1.0.0f-r0", - "1.0.0g-r0", - "1.0.0h-r0", - "1.0.1-r0", - "1.0.1a-r0", - "1.0.1b-r0", - "1.0.1c-r0", - "1.0.1c-r1", - "1.0.1c-r2", - "1.0.1c-r3", - "1.0.1d-r0", - "1.0.1d-r1", - "1.0.1e-r0", - "1.0.1e-r1", - "1.0.1e-r2", - "1.0.1e-r3", - "1.0.1e-r4", - "1.0.1e-r5", - "1.0.1e-r6", - "1.0.1e-r7", - "1.0.1f-r0", - "1.0.1g-r0", - "1.0.1g-r1", - "1.0.1g-r2", - "1.0.1g-r3", - "1.0.1h-r0", - "1.0.1i-r0", - "1.0.1i-r1", - "1.0.1i-r2", - "1.0.1i-r3", - "1.0.1j-r0", - "1.0.1k-r0", - "1.0.1l-r0", - "1.0.2-r0", - "1.0.2a-r0", - "1.0.2a-r1", - "1.0.2b-r0", - "1.0.2c-r0", - "1.0.2d-r0", - "1.0.2e-r0", - "1.0.2e-r1", - "1.0.2f-r0", - "1.0.2f-r1", - "1.0.2f-r2", - "1.0.2g-r0", - "1.0.2g-r1", - "1.0.2g-r2", - "1.0.2g-r3", - "1.0.2h-r0", - "1.0.2h-r1", - "1.0.2h-r2", - "1.0.2h-r3", - "1.0.2h-r4", - "1.0.2i-r0", - "1.0.2j-r0", - "1.0.2j-r1", - "1.0.2j-r2", - "1.0.2k-r0", - "1.0.2l-r0", - "1.0.2m-r0", - "1.0.2n-r0", - "1.0.2o-r0", - "1.0.2o-r1", - "1.0.2p-r0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] }, { "package": { @@ -700,102 +126,7 @@ } ] } - ], - "versions": [ - "0.9.8i-r0", - "0.9.8j-r0", - "0.9.8k-r0", - "0.9.8k-r1", - "0.9.8k-r2", - "0.9.8k-r3", - "0.9.8k-r4", - "0.9.8k-r5", - "0.9.8k-r6", - "0.9.8k-r7", - "0.9.8l-r0", - "0.9.8l-r1", - "0.9.8m-r0", - "0.9.8n-r0", - "0.9.8n-r1", - "1.0.0-r0", - "1.0.0a-r0", - "1.0.0a-r1", - "1.0.0a-r2", - "1.0.0a-r3", - "1.0.0a-r4", - "1.0.0b-r0", - "1.0.0c-r0", - "1.0.0d-r0", - "1.0.0e-r0", - "1.0.0f-r0", - "1.0.0g-r0", - "1.0.0h-r0", - "1.0.1-r0", - "1.0.1a-r0", - "1.0.1b-r0", - "1.0.1c-r0", - "1.0.1c-r1", - "1.0.1c-r2", - "1.0.1c-r3", - "1.0.1d-r0", - "1.0.1d-r1", - "1.0.1e-r0", - "1.0.1e-r1", - "1.0.1e-r2", - "1.0.1e-r3", - "1.0.1e-r4", - "1.0.1e-r5", - "1.0.1e-r6", - "1.0.1e-r7", - "1.0.1f-r0", - "1.0.1g-r0", - "1.0.1g-r1", - "1.0.1g-r2", - "1.0.1g-r3", - "1.0.1h-r0", - "1.0.1i-r0", - "1.0.1i-r1", - "1.0.1i-r2", - "1.0.1i-r3", - "1.0.1j-r0", - "1.0.1k-r0", - "1.0.1l-r0", - "1.0.2-r0", - "1.0.2a-r0", - "1.0.2a-r1", - "1.0.2b-r0", - "1.0.2c-r0", - "1.0.2d-r0", - "1.0.2e-r0", - "1.0.2e-r1", - "1.0.2f-r0", - "1.0.2f-r1", - "1.0.2f-r2", - "1.0.2g-r0", - "1.0.2g-r1", - "1.0.2g-r2", - "1.0.2g-r3", - "1.0.2h-r0", - "1.0.2h-r1", - "1.0.2h-r2", - "1.0.2h-r3", - "1.0.2h-r4", - "1.0.2i-r0", - "1.0.2j-r0", - "1.0.2j-r1", - "1.0.2j-r2", - "1.0.2k-r0", - "1.0.2l-r0", - "1.0.2m-r0", - "1.0.2n-r0", - "1.0.2o-r0", - "1.0.2o-r1", - "1.0.2o-r2", - "1.0.2p-r0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] }, { "ranges": [ @@ -810,31 +141,179 @@ "fixed": "03b825811ed4af0addcdf6e75bacb3dc1c4c5940" } ] + }, + { + "type": "GIT", + "repo": "https://github.com/openssl/openssl", + "events": [ + { + "introduced": "e818b74be2170fbe957a07b0da4401c2b694b3b8" + }, + { + "fixed": "e818b74be2170fbe957a07b0da4401c2b694b3b8" + }, + { + "introduced": "7ea5bd2b52d0e81eaef3d109b3b12545306f201c" + } + ] } - ], - "versions": [ - "v10.0.0", - "v10.1.0", - "v10.2.0", - "v10.2.1", - "v10.3.0", - "v10.4.0", - "v10.4.1", - "v10.5.0", - "v10.6.0", - "v10.7.0", - "v10.8.0" - ], - "database_specific": { - "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2018-5407.json" - } + ] } ], - "schema_version": "1.6.0", - "severity": [ + "references": [ { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:H/I:N/A:N" + "type": "ADVISORY", + "url": "http://www.securityfocus.com/bid/105897" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0483" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0651" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:0652" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:2125" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3929" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3931" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3932" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3933" + }, + { + "type": "ADVISORY", + "url": "https://access.redhat.com/errata/RHSA-2019:3935" + }, + { + "type": "ADVISORY", + "url": "https://security.gentoo.org/glsa/201903-10" + }, + { + "type": "ADVISORY", + "url": "https://security.netapp.com/advisory/ntap-20181126-0001/" + }, + { + "type": "ADVISORY", + "url": "https://www.debian.org/security/2018/dsa-4348" + }, + { + "type": "ADVISORY", + "url": "https://www.debian.org/security/2018/dsa-4355" + }, + { + "type": "ADVISORY", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "ADVISORY", + "url": "https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" + }, + { + "type": "ADVISORY", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujan2019-5072801.html" + }, + { + "type": "ADVISORY", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujul2019-5072835.html" + }, + { + "type": "ARTICLE", + "url": "https://lists.debian.org/debian-lts-announce/2018/11/msg00024.html" + }, + { + "type": "ARTICLE", + "url": "https://nodejs.org/en/blog/vulnerability/november-2018-security-releases/" + }, + { + "type": "EVIDENCE", + "url": "https://github.com/bbbrumley/portsmash" + }, + { + "type": "EVIDENCE", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/security-alerts/cpuapr2020.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/security-alerts/cpujan2020.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujan2019-5072801.html" + }, + { + "type": "FIX", + "url": "https://www.oracle.com/technetwork/security-advisory/cpujul2019-5072835.html" + }, + { + "type": "WEB", + "url": "http://www.securityfocus.com/bid/105897" + }, + { + "type": "WEB", + "url": "https://eprint.iacr.org/2018/1060.pdf" + }, + { + "type": "WEB", + "url": "https://github.com/bbbrumley/portsmash" + }, + { + "type": "WEB", + "url": "https://lists.debian.org/debian-lts-announce/2018/11/msg00024.html" + }, + { + "type": "WEB", + "url": "https://support.f5.com/csp/article/K49711130?utm_source=f5support\u0026amp%3Butm_medium=RSS" + }, + { + "type": "WEB", + "url": "https://usn.ubuntu.com/3840-1/" + }, + { + "type": "WEB", + "url": "https://www.exploit-db.com/exploits/45785/" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpuapr2020.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujan2020.html" + }, + { + "type": "WEB", + "url": "https://www.tenable.com/security/tns-2018-16" + }, + { + "type": "WEB", + "url": "https://www.tenable.com/security/tns-2018-17" } - ] + ], + "modified": "2024-05-29T23:34:56Z", + "published": "2018-11-15T21:29:00Z" } From dbed62ee1f01825da5a78cf07a5d7d64b1cace17 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 31 Jul 2024 04:51:26 +0000 Subject: [PATCH 10/32] Add package-based checks Add checks for package and package version existence. Add end-to-end support for two ecosystems: PyPI and Go --- tools/osv-linter/go.mod | 1 + tools/osv-linter/go.sum | 2 + tools/osv-linter/internal/checks/checks.go | 12 ++ tools/osv-linter/internal/checks/packages.go | 103 +++++++++ tools/osv-linter/internal/checks/ranges.go | 7 +- .../osv-linter/internal/helpers/ecosystems.go | 203 ++++++++++++++++++ tools/osv-linter/internal/helpers/utility.go | 70 ++++++ .../test_data/GHSA-9v2f-6vcg-3hgv.json | 47 ++++ tools/osv-linter/test_data/GO-2020-0001.json | 1 + tools/osv-linter/test_data/GO-2024-2963.json | 1 + tools/osv-linter/test_data/PYSEC-2023-74.json | 134 ++++++++++++ .../nopackage-GHSA-9v2f-6vcg-3hgv.json | 47 ++++ 12 files changed, 622 insertions(+), 6 deletions(-) create mode 100644 tools/osv-linter/internal/checks/packages.go create mode 100644 tools/osv-linter/internal/helpers/ecosystems.go create mode 100644 tools/osv-linter/internal/helpers/utility.go create mode 100644 tools/osv-linter/test_data/GHSA-9v2f-6vcg-3hgv.json create mode 100644 tools/osv-linter/test_data/GO-2020-0001.json create mode 100644 tools/osv-linter/test_data/GO-2024-2963.json create mode 100644 tools/osv-linter/test_data/PYSEC-2023-74.json create mode 100644 tools/osv-linter/test_data/nopackage-GHSA-9v2f-6vcg-3hgv.json diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index ca3134bf..604c35b4 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -4,6 +4,7 @@ go 1.21.10 require ( github.com/google/go-cmp v0.6.0 + github.com/sethvargo/go-retry v0.2.4 github.com/tidwall/gjson v1.17.1 github.com/urfave/cli/v2 v2.27.2 ) diff --git a/tools/osv-linter/go.sum b/tools/osv-linter/go.sum index bdf5c226..934de2f3 100644 --- a/tools/osv-linter/go.sum +++ b/tools/osv-linter/go.sum @@ -4,6 +4,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= +github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index c431ab4f..1b70fd56 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -81,6 +81,16 @@ var Collections = []CheckCollectionDef{ { Name: "ALL", Description: "all checks currently defined", + Checks: []*CheckDef{ + CheckRangeHasIntroducedEvent, + CheckRangeIsDistinct, + CheckPackageExists, + CheckPackageVersionsExist, + }, + }, + { + Name: "offline", + Description: "checks that do not have remote data dependencies", Checks: []*CheckDef{ CheckRangeHasIntroducedEvent, CheckRangeIsDistinct, @@ -92,6 +102,8 @@ var Collections = []CheckCollectionDef{ Checks: []*CheckDef{ CheckRangeHasIntroducedEvent, CheckRangeIsDistinct, + CheckPackageExists, + CheckPackageVersionsExist, }, }, } diff --git a/tools/osv-linter/internal/checks/packages.go b/tools/osv-linter/internal/checks/packages.go new file mode 100644 index 00000000..c844f6ac --- /dev/null +++ b/tools/osv-linter/internal/checks/packages.go @@ -0,0 +1,103 @@ +package checks + +import ( + "fmt" + + "github.com/ossf/osv-schema/linter/internal/helpers" + "github.com/tidwall/gjson" +) + +var CheckPackageExists = &CheckDef{ + Code: "P0001", + Name: "package-exists", + Description: "package exists in ecosystem's registry", + Check: PackageExists, +} + +// PackageExists checks the package exists in the registry for that ecosystem. +func PackageExists(json *gjson.Result) (findings []CheckError) { + affectedEntries := json.Get(`affected`) + + // Examine each entry: + // for ones for packages, on a per-package basis + affectedEntries.ForEach(func(key, value gjson.Result) bool { + // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. + maybePackage := value.Get(`package`) + if !maybePackage.Exists() { + return true // keep iterating (over affected entries) + } + ecosystem := value.Get(`package.ecosystem`) + pkg := value.Get(`package.name`) + if !helpers.PackageExistsInEcosystem(pkg.String(), ecosystem.String()) { + findings = append(findings, CheckError{Message: fmt.Sprintf("package not found: %q", pkg)}) + } + return true // keep iterating (over affected entries) + }) + return findings +} + +var CheckPackageVersionsExist = &CheckDef{ + Code: "P0002", + Name: "package-versions-exist", + Description: "package versions exist in ecosystem's registry", + Check: PackageVersionsExist, +} + +// PackageVersionsExist checks the package versions exist in the registry for that ecosystem. +func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { + affectedEntries := json.Get(`affected`) + + // Examine each affected entry: + // for ones for packages, on a per-package basis + affectedEntries.ForEach(func(key, value gjson.Result) bool { + // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. + maybePackage := value.Get(`package`) + if !maybePackage.Exists() { + return true // keep iterating (over affected entries) + } + ecosystem := value.Get(`package.ecosystem`).String() + pkg := value.Get(`package.name`).String() + versionsToCheck := []string{} + // Examine versions in ranges. + maybeRanges := value.Get(`ranges`) + maybeRanges.ForEach(func(key, value gjson.Result) bool { + rangeType := value.Get(`type`).String() + if rangeType == "GIT" { + return true // keep iterating (over ranges) + } + events := value.Get(`events`) + events.ForEach(func(key, value gjson.Result) bool { + // Collect all the introduced values. + result := value.Get(`introduced`) + if result.Exists() && result.String() != "0" { + versionsToCheck = append(versionsToCheck, result.String()) + } + // Collect all the fixed/last_affected values. + result = value.Get(`fixed`) + if result.Exists() { + versionsToCheck = append(versionsToCheck, result.String()) + } + result = value.Get(`last_affected`) + if result.Exists() { + versionsToCheck = append(versionsToCheck, result.String()) + } + return true // keep iterating (over events) + }) + return true // keep iterating (over ranges) + }) + // Examine versions in versions array. + maybeVersions := value.Get(`versions`) + maybeVersions.ForEach(func(key, value gjson.Result) bool { + versionsToCheck = append(versionsToCheck, value.String()) + return true // keep iterating (over versions) + }) + println(fmt.Sprintf("Checking for: %#v", versionsToCheck)) + err := helpers.PackageVersionsExistInEcosystem(pkg, versionsToCheck, ecosystem) + if err != nil { + findings = append(findings, CheckError{Message: fmt.Sprintf("Failed to find some versions of %s: %#v", pkg, err)}) + } + + return true // keep iterating (over affected entries) + }) + return findings +} diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index 7feed6ce..c5dc02cd 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -33,11 +33,6 @@ var CheckRangeIsDistinct = &CheckDef{ Check: RangeIsDistinct, } -type Range struct { - Beginning string - End string -} - // RangeIsDistinct checks that the introduced and fixed (or last_affected) values differ. // (on a per-repo basis for GIT ranges, and on a per-package basis otherwise) func RangeIsDistinct(json *gjson.Result) (findings []CheckError) { @@ -92,4 +87,4 @@ func RangeIsDistinct(json *gjson.Result) (findings []CheckError) { return true // keep iterating (over affected entries) }) return findings -} +} \ No newline at end of file diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go new file mode 100644 index 00000000..e6bd435f --- /dev/null +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -0,0 +1,203 @@ +package helpers + +import ( + "fmt" + "io" + "net/http" + "slices" + "strings" + + "github.com/tidwall/gjson" +) + +// Dispatcher for ecosystem-specific package existence checking. +func PackageExistsInEcosystem(pkg string, ecosystem string) bool { + switch ecosystem { + case "PyPI": + return PackageExistsInPyPI(pkg) + case "Go": + return PackageExistsInGo(pkg) + } + return false +} + +// Dispatcher for ecosystem-specific package version existence checking. +func PackageVersionsExistInEcosystem(pkg string, versions []string, ecosystem string) error { + switch ecosystem { + case "PyPI": + return PackageVersionsExistInPyPI(pkg, versions) + case "Go": + return PackageVersionsExistInGo(pkg, versions) + } + return fmt.Errorf("unsupported ecosystem: %s", ecosystem) +} + +// Validate the existence of a package in PyPI. +func PackageExistsInPyPI(pkg string) bool { + packageURL := "https://pypi.org/pypi/{package}/json" + + packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + + // This 404's for non-existent packages. + resp, err := Head(packageInstanceURL) + if err != nil { + return false + } + if resp.StatusCode == http.StatusOK { + return true + } + + return false +} + +// Confirm that all specified versions of a package exist in PyPI. +func PackageVersionsExistInPyPI(pkg string, versions []string) error { + packageURL := "https://pypi.org/pypi/{package}/json" + + packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + + // This 404's for non-existent packages. + resp, err := Get(packageInstanceURL) + if err != nil { + return fmt.Errorf("unable to validate package: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unable to validate package: %q for %s", resp.Status, packageInstanceURL) + } + + // Parse the known versions from the JSON. + respJSON, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("unable to retrieve JSON for %q: %v", pkg, err) + } + // Fetch all known versions of package. + versionsInPyPy := []string{} + releases := gjson.GetBytes(respJSON, `releases.@keys`) + releases.ForEach(func(key, value gjson.Result) bool { + versionsInPyPy = append(versionsInPyPy, value.String()) + return true // keep iterating. + }) + // Determine which referenced versions are missing. + versionsMissing := []string{} + for _, versionToCheckFor := range versions { + if slices.Contains(versionsInPyPy, versionToCheckFor) { + continue + } + versionsMissing = append(versionsMissing, versionToCheckFor) + } + if len(versionsMissing) > 0 { + return fmt.Errorf("failed to find %#v for %q", versionsMissing, pkg) + } + + return nil +} + +// Validate the existence of a package in Go. +func PackageExistsInGo(pkg string) bool { + packageURL := "https://proxy.golang.org/{package}/@v/list" + + // Of course the Go runtime exists :-) + if pkg == "stdlib" { + return true + } + + packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + + // This 404's for non-existent packages. + resp, err := Head(packageInstanceURL) + if err != nil { + return false + } + if resp.StatusCode == http.StatusOK { + return true + } + + return false +} + +// Confirm that all specified versions of a package exist in Go. +func PackageVersionsExistInGo(pkg string, versions []string) error { + packageURL := "https://proxy.golang.org/{package}/@v/list" + + if pkg == "stdlib" { + return GoVersionsExist(versions) + } + + packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + + // This 404's for non-existent packages. + resp, err := Get(packageInstanceURL) + if err != nil { + return fmt.Errorf("unable to validate package: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unable to validate package: %q for %s", resp.Status, packageInstanceURL) + } + + // Load the known versions from the list provided. + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("unable to retrieve versions for for %q: %v", pkg, err) + } + // Fetch all known versions of package. + versionsInGo := strings.Split(string(respBytes), "\n") + println(fmt.Sprintf("%#v", versionsInGo)) + + // Determine which referenced versions are missing. + versionsMissing := []string{} + for _, versionToCheckFor := range versions { + if slices.Contains(versionsInGo, versionToCheckFor) { + continue + } + versionsMissing = append(versionsMissing, versionToCheckFor) + } + if len(versionsMissing) > 0 { + return fmt.Errorf("failed to find %#v for %q", versionsMissing, pkg) + } + + return nil +} + +// Confirm that all specified versions of Go exist. +func GoVersionsExist(versions []string) error { + URL := "https://go.dev/dl/?mode=json&include=all" + + resp, err := Get(URL) + if err != nil { + return fmt.Errorf("unable to validate Go versions: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unable to validate package: %q for %s", resp.Status, URL) + } + + // Fetch all known versions of Go. + // Parse the known versions from the JSON. + respJSON, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("unable to retrieve JSON for Go: %v", err) + } + // Fetch all known versions of package. + GoVersions := []string{} + releases := gjson.GetBytes(respJSON, `#.version`) + releases.ForEach(func(key, value gjson.Result) bool { + GoVersions = append(GoVersions, value.String()) + return true // keep iterating. + }) + + // Determine which referenced versions are missing. + versionsMissing := []string{} + for _, versionToCheckFor := range versions { + if slices.Contains(GoVersions, "go"+versionToCheckFor) { + continue + } + versionsMissing = append(versionsMissing, versionToCheckFor) + } + if len(versionsMissing) > 0 { + return fmt.Errorf("failed to find %#v for Go", versionsMissing) + } + + return nil +} diff --git a/tools/osv-linter/internal/helpers/utility.go b/tools/osv-linter/internal/helpers/utility.go new file mode 100644 index 00000000..7f061398 --- /dev/null +++ b/tools/osv-linter/internal/helpers/utility.go @@ -0,0 +1,70 @@ +package helpers + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/sethvargo/go-retry" +) + +// Make a HTTP GET request for url and retry 3 times, with an exponential backoff. +func Get(url string) (resp *http.Response, err error) { + backoff := retry.NewExponential(1 * time.Second) + if err := retry.Do(context.Background(), retry.WithMaxRetries(3, backoff), func(ctx context.Context) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + r, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + // defer r.Body.Close() + + switch r.StatusCode / 100 { + case 4: + return fmt.Errorf("bad response: %v", r.StatusCode) + case 5: + return retry.RetryableError(fmt.Errorf("bad response: %v", r.StatusCode)) + default: + resp = r + return nil + } + }); err != nil { + return nil, fmt.Errorf("fail: %q: %v", url, err) + } + return resp, err +} + +// Make a HTTP HEAD request for url and retry 3 times, with an exponential backoff. +func Head(url string) (resp *http.Response, err error) { + backoff := retry.NewExponential(1 * time.Second) + if err := retry.Do(context.Background(), retry.WithMaxRetries(3, backoff), func(ctx context.Context) error { + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return err + } + + r, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer r.Body.Close() + + switch r.StatusCode / 100 { + case 4: + return fmt.Errorf("bad response: %v", r.StatusCode) + case 5: + return retry.RetryableError(fmt.Errorf("bad response: %v", r.StatusCode)) + default: + resp = r + return nil + } + }); err != nil { + return nil, fmt.Errorf("fail: %q: %v", url, err) + } + return resp, err +} diff --git a/tools/osv-linter/test_data/GHSA-9v2f-6vcg-3hgv.json b/tools/osv-linter/test_data/GHSA-9v2f-6vcg-3hgv.json new file mode 100644 index 00000000..19e4ec10 --- /dev/null +++ b/tools/osv-linter/test_data/GHSA-9v2f-6vcg-3hgv.json @@ -0,0 +1,47 @@ +{ + "schema_version": "1.4.0", + "id": "GHSA-9v2f-6vcg-3hgv", + "modified": "2024-07-03T20:05:21Z", + "published": "2024-07-01T21:31:15Z", + "aliases": [ + "CVE-2024-39236" + ], + "summary": "Gradio was discovered to contain a code injection vulnerability via the component /gradio/component_meta.py", + "details": "Gradio v4.36.1 was discovered to contain a code injection vulnerability via the component /gradio/component_meta.py. This vulnerability is triggered via a crafted input.", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + } + ], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "Gradio" + }, + "versions": [ + "4.36.1" + ] + } + ], + "references": [ + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-39236" + }, + { + "type": "WEB", + "url": "https://github.com/Aaron911/PoC/blob/main/Gradio.md" + } + ], + "database_specific": { + "cwe_ids": [ + "CWE-94" + ], + "severity": "CRITICAL", + "github_reviewed": true, + "github_reviewed_at": "2024-07-01T22:13:35Z", + "nvd_published_at": "2024-07-01T19:15:05Z" + } +} \ No newline at end of file diff --git a/tools/osv-linter/test_data/GO-2020-0001.json b/tools/osv-linter/test_data/GO-2020-0001.json new file mode 100644 index 00000000..cff065af --- /dev/null +++ b/tools/osv-linter/test_data/GO-2020-0001.json @@ -0,0 +1 @@ +{"schema_version":"1.3.1","id":"GO-2020-0001","modified":"2024-05-20T16:03:47Z","published":"2021-04-14T20:04:52Z","aliases":["CVE-2020-36567","GHSA-6vm3-jj99-7229"],"summary":"Arbitrary log line injection in github.com/gin-gonic/gin","details":"The default Formatter for the Logger middleware (LoggerConfig.Formatter), which is included in the Default engine, allows attackers to inject arbitrary log entries by manipulating the request path.","affected":[{"package":{"name":"github.com/gin-gonic/gin","ecosystem":"Go"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"1.6.0"}]}],"ecosystem_specific":{"imports":[{"path":"github.com/gin-gonic/gin","symbols":["Default","Logger","LoggerWithConfig","LoggerWithFormatter","LoggerWithWriter"]}]}}],"references":[{"type":"FIX","url":"https://github.com/gin-gonic/gin/pull/2237"},{"type":"FIX","url":"https://github.com/gin-gonic/gin/commit/a71af9c144f9579f6dbe945341c1df37aaf09c0d"}],"credits":[{"name":"@thinkerou \u003cthinkerou@gmail.com\u003e"}],"database_specific":{"url":"https://pkg.go.dev/vuln/GO-2020-0001","review_status":"REVIEWED"}} \ No newline at end of file diff --git a/tools/osv-linter/test_data/GO-2024-2963.json b/tools/osv-linter/test_data/GO-2024-2963.json new file mode 100644 index 00000000..2f70fc55 --- /dev/null +++ b/tools/osv-linter/test_data/GO-2024-2963.json @@ -0,0 +1 @@ +{"schema_version":"1.3.1","id":"GO-2024-2963","modified":"2024-07-02T20:11:00Z","published":"2024-07-02T20:11:00Z","aliases":["CVE-2024-24791"],"summary":"Denial of service due to improper 100-continue handling in net/http","details":"The net/http HTTP/1.1 client mishandled the case where a server responds to a request with an \"Expect: 100-continue\" header with a non-informational (200 or higher) status. This mishandling could leave a client connection in an invalid state, where the next request sent on the connection will fail.\n\nAn attacker sending a request to a net/http/httputil.ReverseProxy proxy can exploit this mishandling to cause a denial of service by sending \"Expect: 100-continue\" requests which elicit a non-informational response from the backend. Each such request leaves the proxy with an invalid connection, and causes one subsequent request using that connection to fail.","affected":[{"package":{"name":"stdlib","ecosystem":"Go"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"1.21.12"},{"introduced":"1.22.0-0"},{"fixed":"1.22.5"}]}],"ecosystem_specific":{"imports":[{"path":"net/http","symbols":["Client.CloseIdleConnections","Client.Do","Client.Get","Client.Head","Client.Post","Client.PostForm","Get","Head","Post","PostForm","Transport.CancelRequest","Transport.CloseIdleConnections","Transport.RoundTrip","persistConn.readResponse"]}]}}],"references":[{"type":"FIX","url":"https://go.dev/cl/591255"},{"type":"REPORT","url":"https://go.dev/issue/67555"},{"type":"WEB","url":"https://groups.google.com/g/golang-dev/c/t0rK-qHBqzY/m/6MMoAZkMAgAJ"}],"credits":[{"name":"Geoff Franks"}],"database_specific":{"url":"https://pkg.go.dev/vuln/GO-2024-2963","review_status":"REVIEWED"}} \ No newline at end of file diff --git a/tools/osv-linter/test_data/PYSEC-2023-74.json b/tools/osv-linter/test_data/PYSEC-2023-74.json new file mode 100644 index 00000000..d8ffb731 --- /dev/null +++ b/tools/osv-linter/test_data/PYSEC-2023-74.json @@ -0,0 +1,134 @@ +{ + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "requests", + "purl": "pkg:pypi/requests" + }, + "ranges": [ + { + "events": [ + { + "introduced": "0" + }, + { + "fixed": "74ea7cf7a6a27a4eeb2ae24e162bcc942a6706d5" + } + ], + "repo": "https://github.com/psf/requests", + "type": "GIT" + }, + { + "events": [ + { + "introduced": "2.3.0" + }, + { + "fixed": "2.31.0" + } + ], + "type": "ECOSYSTEM" + } + ], + "versions": [ + "2.10.0", + "2.11.0", + "2.11.1", + "2.12.0", + "2.12.1", + "2.12.2", + "2.12.3", + "2.12.4", + "2.12.5", + "2.13.0", + "2.14.0", + "2.14.1", + "2.14.2", + "2.15.0", + "2.15.1", + "2.16.0", + "2.16.1", + "2.16.2", + "2.16.3", + "2.16.4", + "2.16.5", + "2.17.0", + "2.17.1", + "2.17.2", + "2.17.3", + "2.18.0", + "2.18.1", + "2.18.2", + "2.18.3", + "2.18.4", + "2.19.0", + "2.19.1", + "2.20.0", + "2.20.1", + "2.21.0", + "2.22.0", + "2.23.0", + "2.24.0", + "2.25.0", + "2.25.1", + "2.26.0", + "2.27.0", + "2.27.1", + "2.28.0", + "2.28.1", + "2.28.2", + "2.29.0", + "2.3.0", + "2.30.0", + "2.4.0", + "2.4.1", + "2.4.2", + "2.4.3", + "2.5.0", + "2.5.1", + "2.5.2", + "2.5.3", + "2.6.0", + "2.6.1", + "2.6.2", + "2.7.0", + "2.8.0", + "2.8.1", + "2.9.0", + "2.9.1", + "2.9.2" + ] + } + ], + "aliases": [ + "CVE-2023-32681", + "GHSA-j8r2-6x86-q33q" + ], + "details": "Requests is a HTTP library. Since Requests 2.3.0, Requests has been leaking Proxy-Authorization headers to destination servers when redirected to an HTTPS endpoint. This is a product of how we use `rebuild_proxies` to reattach the `Proxy-Authorization` header to requests. For HTTP connections sent through the tunnel, the proxy will identify the header in the request itself and remove it prior to forwarding to the destination server. However when sent over HTTPS, the `Proxy-Authorization` header must be sent in the CONNECT request as the proxy has no visibility into the tunneled request. This results in Requests forwarding proxy credentials to the destination server unintentionally, allowing a malicious actor to potentially exfiltrate sensitive information. This issue has been patched in version 2.31.0.\n\n", + "id": "PYSEC-2023-74", + "modified": "2023-06-05T01:13:00.534973Z", + "published": "2023-05-26T18:15:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://github.com/psf/requests/security/advisories/GHSA-j8r2-6x86-q33q" + }, + { + "type": "WEB", + "url": "https://github.com/psf/requests/releases/tag/v2.31.0" + }, + { + "type": "FIX", + "url": "https://github.com/psf/requests/commit/74ea7cf7a6a27a4eeb2ae24e162bcc942a6706d5" + }, + { + "type": "ARTICLE", + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/AW7HNFGYP44RT3DUDQXG2QT3OEV2PJ7Y/" + }, + { + "type": "WEB", + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/AW7HNFGYP44RT3DUDQXG2QT3OEV2PJ7Y/" + } + ] +} diff --git a/tools/osv-linter/test_data/nopackage-GHSA-9v2f-6vcg-3hgv.json b/tools/osv-linter/test_data/nopackage-GHSA-9v2f-6vcg-3hgv.json new file mode 100644 index 00000000..9ff226c7 --- /dev/null +++ b/tools/osv-linter/test_data/nopackage-GHSA-9v2f-6vcg-3hgv.json @@ -0,0 +1,47 @@ +{ + "schema_version": "1.4.0", + "id": "GHSA-9v2f-6vcg-3hgv", + "modified": "2024-07-03T20:05:21Z", + "published": "2024-07-01T21:31:15Z", + "aliases": [ + "CVE-2024-39236" + ], + "summary": "Gradio was discovered to contain a code injection vulnerability via the component /gradio/component_meta.py", + "details": "Gradio v4.36.1 was discovered to contain a code injection vulnerability via the component /gradio/component_meta.py. This vulnerability is triggered via a crafted input.", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + } + ], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "Gradi0" + }, + "versions": [ + "4.36.1" + ] + } + ], + "references": [ + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-39236" + }, + { + "type": "WEB", + "url": "https://github.com/Aaron911/PoC/blob/main/Gradio.md" + } + ], + "database_specific": { + "cwe_ids": [ + "CWE-94" + ], + "severity": "CRITICAL", + "github_reviewed": true, + "github_reviewed_at": "2024-07-01T22:13:35Z", + "nvd_published_at": "2024-07-01T19:15:05Z" + } +} From 26ccb3c848367e38205879be875222165aa5e7df Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 31 Jul 2024 05:48:42 +0000 Subject: [PATCH 11/32] Package check optimisation and improvements - Normalize ecosystems with colons in them down to their base - Cache the existence/nonexistence of a package (in a normalized ecosystem) to reduce duplicate network checks - Correct the test data for CVE-2018-5407 to be the current live record without overlapping ranges present (this shouldn't fail range validation) --- tools/osv-linter/internal/checks/packages.go | 46 +++++++++++++++++-- tools/osv-linter/test_data/CVE-2018-5407.json | 19 ++------ 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/tools/osv-linter/internal/checks/packages.go b/tools/osv-linter/internal/checks/packages.go index c844f6ac..649ddd72 100644 --- a/tools/osv-linter/internal/checks/packages.go +++ b/tools/osv-linter/internal/checks/packages.go @@ -2,6 +2,8 @@ package checks import ( "fmt" + "slices" + "strings" "github.com/ossf/osv-schema/linter/internal/helpers" "github.com/tidwall/gjson" @@ -18,6 +20,9 @@ var CheckPackageExists = &CheckDef{ func PackageExists(json *gjson.Result) (findings []CheckError) { affectedEntries := json.Get(`affected`) + knownExistent := make(map[string][]string) + knownNonexistent := make(map[string][]string) + // Examine each entry: // for ones for packages, on a per-package basis affectedEntries.ForEach(func(key, value gjson.Result) bool { @@ -26,10 +31,39 @@ func PackageExists(json *gjson.Result) (findings []CheckError) { if !maybePackage.Exists() { return true // keep iterating (over affected entries) } - ecosystem := value.Get(`package.ecosystem`) - pkg := value.Get(`package.name`) - if !helpers.PackageExistsInEcosystem(pkg.String(), ecosystem.String()) { - findings = append(findings, CheckError{Message: fmt.Sprintf("package not found: %q", pkg)}) + // Normalize ecosystems with a colon to their base. + // e.g. "Alpine:v3.5" -> "Alpine" + ecosystem := strings.Split(value.Get(`package.ecosystem`).String(), ":")[0] + pkg := value.Get(`package.name`).String() + + // Avoid unnecessary network traffic for repeat packages. + if _, ok := knownExistent[ecosystem]; ok { + if slices.Contains(knownExistent[ecosystem], pkg) { + return true // keep iterating (over affected entries) + } + } + if _, ok := knownNonexistent[ecosystem]; ok { + if slices.Contains(knownNonexistent[ecosystem], pkg) { + // Don't add repeat findings for the same package. + return true // keep iterating (over affected entries) + } + } + // Not cached, determine existence. + if !helpers.PackageExistsInEcosystem(pkg, ecosystem) { + findings = append(findings, CheckError{Message: fmt.Sprintf("package %q not found", pkg)}) + _, ok := knownNonexistent[ecosystem] + if ok { + knownNonexistent[ecosystem] = append(knownNonexistent[ecosystem], pkg) + } else { + knownNonexistent[ecosystem] = []string{pkg} + } + } else { + _, ok := knownExistent[ecosystem] + if ok { + knownExistent[ecosystem] = append(knownExistent[ecosystem], pkg) + } else { + knownExistent[ecosystem] = []string{pkg} + } } return true // keep iterating (over affected entries) }) @@ -55,7 +89,9 @@ func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { if !maybePackage.Exists() { return true // keep iterating (over affected entries) } - ecosystem := value.Get(`package.ecosystem`).String() + // Normalize ecosystems with a colon to their base. + // e.g. "Alpine:v3.5" -> "Alpine" + ecosystem := strings.Split(value.Get(`package.ecosystem`).String(), ":")[0] pkg := value.Get(`package.name`).String() versionsToCheck := []string{} // Examine versions in ranges. diff --git a/tools/osv-linter/test_data/CVE-2018-5407.json b/tools/osv-linter/test_data/CVE-2018-5407.json index e9ad70aa..742e8d83 100644 --- a/tools/osv-linter/test_data/CVE-2018-5407.json +++ b/tools/osv-linter/test_data/CVE-2018-5407.json @@ -141,21 +141,6 @@ "fixed": "03b825811ed4af0addcdf6e75bacb3dc1c4c5940" } ] - }, - { - "type": "GIT", - "repo": "https://github.com/openssl/openssl", - "events": [ - { - "introduced": "e818b74be2170fbe957a07b0da4401c2b694b3b8" - }, - { - "fixed": "e818b74be2170fbe957a07b0da4401c2b694b3b8" - }, - { - "introduced": "7ea5bd2b52d0e81eaef3d109b3b12545306f201c" - } - ] } ] } @@ -312,6 +297,10 @@ { "type": "WEB", "url": "https://www.tenable.com/security/tns-2018-17" + }, + { + "type": "ADVISORY", + "url": "https://security.alpinelinux.org/vuln/CVE-2018-5407" } ], "modified": "2024-05-29T23:34:56Z", From a3dfb9e2377f1cfd78534369d337700b34c8b1de Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 31 Jul 2024 06:20:57 +0000 Subject: [PATCH 12/32] Make RangeHasIntroducedEvent tolerate records without ranges It is valid to not have any range at all, as seen in the likes of GHSA-9v2f-6vcg-3hgv, which was being flagged incorrectly. --- tools/osv-linter/internal/checks/ranges.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index c5dc02cd..22cd50e7 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -16,6 +16,12 @@ var CheckRangeHasIntroducedEvent = &CheckDef{ // RangeHasIntroducedEvent checks for missing 'introduced' objects in events. func RangeHasIntroducedEvent(json *gjson.Result) (findings []CheckError) { + // It is valid to not have any ranges. + ranges := json.Get(`affected.#(ranges)`) + if !ranges.Exists() { + return nil + } + result := json.Get(`affected.#(ranges.#(events.#(introduced)))`) if !result.Exists() { From 5da5d87b3b79782a01d865f89fd37fdea9ac9cfc Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 31 Jul 2024 06:55:48 +0000 Subject: [PATCH 13/32] Add check that validates Purls Remove some println() debugging --- tools/osv-linter/go.mod | 1 + tools/osv-linter/go.sum | 2 ++ tools/osv-linter/internal/checks/checks.go | 3 ++ tools/osv-linter/internal/checks/packages.go | 36 ++++++++++++++++++- .../osv-linter/internal/helpers/ecosystems.go | 1 - 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index 604c35b4..c49907b1 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -4,6 +4,7 @@ go 1.21.10 require ( github.com/google/go-cmp v0.6.0 + github.com/package-url/packageurl-go v0.1.3 github.com/sethvargo/go-retry v0.2.4 github.com/tidwall/gjson v1.17.1 github.com/urfave/cli/v2 v2.27.2 diff --git a/tools/osv-linter/go.sum b/tools/osv-linter/go.sum index 934de2f3..3cdecc42 100644 --- a/tools/osv-linter/go.sum +++ b/tools/osv-linter/go.sum @@ -2,6 +2,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs= +github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index 1b70fd56..e3d69ab7 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -86,6 +86,7 @@ var Collections = []CheckCollectionDef{ CheckRangeIsDistinct, CheckPackageExists, CheckPackageVersionsExist, + CheckPackagePurlValid, }, }, { @@ -94,6 +95,7 @@ var Collections = []CheckCollectionDef{ Checks: []*CheckDef{ CheckRangeHasIntroducedEvent, CheckRangeIsDistinct, + CheckPackagePurlValid, }, }, { @@ -104,6 +106,7 @@ var Collections = []CheckCollectionDef{ CheckRangeIsDistinct, CheckPackageExists, CheckPackageVersionsExist, + CheckPackagePurlValid, }, }, } diff --git a/tools/osv-linter/internal/checks/packages.go b/tools/osv-linter/internal/checks/packages.go index 649ddd72..274b210e 100644 --- a/tools/osv-linter/internal/checks/packages.go +++ b/tools/osv-linter/internal/checks/packages.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/ossf/osv-schema/linter/internal/helpers" + "github.com/package-url/packageurl-go" "github.com/tidwall/gjson" ) @@ -127,7 +128,6 @@ func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { versionsToCheck = append(versionsToCheck, value.String()) return true // keep iterating (over versions) }) - println(fmt.Sprintf("Checking for: %#v", versionsToCheck)) err := helpers.PackageVersionsExistInEcosystem(pkg, versionsToCheck, ecosystem) if err != nil { findings = append(findings, CheckError{Message: fmt.Sprintf("Failed to find some versions of %s: %#v", pkg, err)}) @@ -137,3 +137,37 @@ func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { }) return findings } + +var CheckPackagePurlValid = &CheckDef{ + Code: "P0003", + Name: "package-purl-valid", + Description: "package purl validates", + Check: PackagePurlValid, +} + +// PackagePurlValid checks the package purls validate. +func PackagePurlValid(json *gjson.Result) (findings []CheckError) { + affectedEntries := json.Get(`affected`) + + // Examine each affected entry: + // for ones for packages, on a per-package basis + affectedEntries.ForEach(func(key, value gjson.Result) bool { + // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. + maybePackage := value.Get(`package`) + if !maybePackage.Exists() { + return true // keep iterating (over affected entries) + } + purl := value.Get(`package.purl`) + if !purl.Exists() { + return true // keep iterating (over affected entries) + } + + _, err := packageurl.FromString(purl.String()) + if err != nil { + findings = append(findings, CheckError{Message: fmt.Sprintf("Invalid Purl %q: %#v", purl.String(), err)}) + } + + return true // keep iterating (over affected entries) + }) + return findings +} diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index e6bd435f..e878dfce 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -143,7 +143,6 @@ func PackageVersionsExistInGo(pkg string, versions []string) error { } // Fetch all known versions of package. versionsInGo := strings.Split(string(respBytes), "\n") - println(fmt.Sprintf("%#v", versionsInGo)) // Determine which referenced versions are missing. versionsMissing := []string{} From d1e54315fd3a53d35045d2936ce8945076690a72 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 1 Aug 2024 00:00:35 +0000 Subject: [PATCH 14/32] Force at least GitHub package names to lowercase The Go module proxy seems to not support package names with uppercase in their name. GitHub URLs are known to be case-insensitive, so it's safe to explicitly lowercase these. I dare say it'll be safe to lowercase everything, but I wanted to start conservatively for now. Also treat Go toolchain vulnerabilities the same way as stdlib ones so they aren't flagged as a non-existent package. --- tools/osv-linter/internal/helpers/ecosystems.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index e878dfce..b91dc65d 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -98,10 +98,16 @@ func PackageExistsInGo(pkg string) bool { packageURL := "https://proxy.golang.org/{package}/@v/list" // Of course the Go runtime exists :-) - if pkg == "stdlib" { + if pkg == "stdlib" || pkg == "toolchain" { return true } + // The Go Module Proxy seems to require package names to be lowercase. + // GitHub URLs are known to be case-insensitive. + if strings.HasPrefix(pkg, "github.com/") { + pkg = strings.ToLower(pkg) + } + packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) // This 404's for non-existent packages. @@ -120,10 +126,16 @@ func PackageExistsInGo(pkg string) bool { func PackageVersionsExistInGo(pkg string, versions []string) error { packageURL := "https://proxy.golang.org/{package}/@v/list" - if pkg == "stdlib" { + if pkg == "stdlib" || pkg == "toolchain" { return GoVersionsExist(versions) } + // The Go Module Proxy seems to require package names to be lowercase. + // GitHub URLs are known to be case-insensitive. + if strings.HasPrefix(pkg, "github.com/") { + pkg = strings.ToLower(pkg) + } + packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) // This 404's for non-existent packages. From 0a736762b1ebc2a39c73744dc0651d05856abe98 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Tue, 6 Aug 2024 23:16:05 +0000 Subject: [PATCH 15/32] Fix the operation of a single check Overwrite, don't append so that only the check requested gets run --- tools/osv-linter/internal/helpers/ecosystems.go | 4 ++-- tools/osv-linter/internal/linter.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index b91dc65d..46c925ae 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -165,7 +165,7 @@ func PackageVersionsExistInGo(pkg string, versions []string) error { versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { - return fmt.Errorf("failed to find %#v for %q", versionsMissing, pkg) + return fmt.Errorf("failed to find %+v for %q in %+v", versionsMissing, pkg, versionsInGo) } return nil @@ -207,7 +207,7 @@ func GoVersionsExist(versions []string) error { versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { - return fmt.Errorf("failed to find %#v for Go", versionsMissing) + return fmt.Errorf("failed to find %+v for Go in %+v", versionsMissing, GoVersions) } return nil diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index 17185ef1..13e79bc4 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -79,14 +79,14 @@ func LintCommand(cCtx *cli.Context) error { checksToBeRun = collection.Checks } - // Run just an individual check. + // Run just an individual check, overriding anything discovered from a collection. if code := cCtx.String("check"); code != "" { // Check the requested check exists. check := checks.FromCode(code) if check == nil { return fmt.Errorf("%q is not a valid check", code) } - checksToBeRun = append(checksToBeRun, check) + checksToBeRun = []*checks.CheckDef{check} } perFileFindings := map[string][]checks.CheckError{} From a0e0c49b1329624f7dda022e1ab389323bfb3a77 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 7 Aug 2024 04:27:28 +0000 Subject: [PATCH 16/32] Identify and skip version validation for pseudoversions They don't get returned by the Go proxy. Also support versions with or without the "v" prefix. All existing published Go vulnerabilities with the exception of GO-2024-3012 now pass validation. Signed-off-by: Andrew Pollock --- .../osv-linter/internal/helpers/ecosystems.go | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index 46c925ae..29f3d146 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net/http" + "regexp" "slices" "strings" @@ -122,6 +123,24 @@ func PackageExistsInGo(pkg string) bool { return false } +// isGoPseudoVersion checks if a given version string is a Go pseudo-version, +// including those with pre-release and build metadata segments, +// and handles cases where the pre-release identifier starts with '0.'. +func isGoPseudoVersion(version string) bool { + // Seen in the wild: + // 1.2.0.0 + // 0.5.0-alpha.5.0.20200423152442-f4b650b51dc4 + // 1.0.0-beta + // 1.0.4-0.20180125103619-43913f2f4fbd + // 1.1.10-0.20180427153919-f5cbcbc5cc6f + // 1.16.0-0 + // 2.2.5-rc6.0.20190621200032-0ddffe484adc+incompatible + + // Regular expression to match pseudoversions. + pseudoVersionRegex := regexp.MustCompile(`^(0\.|[0-9]+\.[0-9]+\.)(?:0+|(?:\d+(?:[.-](?:rc)?\d+){0,2})(?:\.(?:0+|(?:\d+(?:[.-]\d+){0,2}))){1,2})([-+].+)?$`) + return pseudoVersionRegex.MatchString(version) +} + // Confirm that all specified versions of a package exist in Go. func PackageVersionsExistInGo(pkg string, versions []string) error { packageURL := "https://proxy.golang.org/{package}/@v/list" @@ -155,11 +174,28 @@ func PackageVersionsExistInGo(pkg string, versions []string) error { } // Fetch all known versions of package. versionsInGo := strings.Split(string(respBytes), "\n") + // It seems that an empty version set is plausible. Unreleased? + // e.g. github.com/nanobox-io/golang-nanoauth + if len(versionsInGo[0]) == 0 { + versionsInGo = []string{} + } + if len(versionsInGo) == 0 { + // TODO: This is warning-level worthy if warnings were a thing... + return nil + } // Determine which referenced versions are missing. versionsMissing := []string{} for _, versionToCheckFor := range versions { - if slices.Contains(versionsInGo, versionToCheckFor) { + // Add pseudo-version to base version mapping here. + // First, detect pseudo-version and skip it. + if isGoPseudoVersion(versionToCheckFor) { + // TODO: Try mapping the pseudo-version to a base version and + // checking for that instead of skipping. + continue + } + // Check for both bare versions and "v"-prefixed versions. + if slices.Contains(versionsInGo, versionToCheckFor) || slices.Contains(versionsInGo, "v"+versionToCheckFor) { continue } versionsMissing = append(versionsMissing, versionToCheckFor) @@ -201,6 +237,10 @@ func GoVersionsExist(versions []string) error { // Determine which referenced versions are missing. versionsMissing := []string{} for _, versionToCheckFor := range versions { + if isGoPseudoVersion(versionToCheckFor) { + // TODO: Try mapping the pseudo-version to a base version instead of skipping. + continue + } if slices.Contains(GoVersions, "go"+versionToCheckFor) { continue } From 313d4c3b4fae2a643fe04da8c1e7c3354b982206 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 01:41:47 +0000 Subject: [PATCH 17/32] fix: remove osv.dev check collection Unnecessary, default to the ALL one. Signed-off-by: Andrew Pollock --- tools/osv-linter/cmd/osv/main.go | 2 +- tools/osv-linter/internal/checks/checks.go | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tools/osv-linter/cmd/osv/main.go b/tools/osv-linter/cmd/osv/main.go index fad4223f..21d0538f 100644 --- a/tools/osv-linter/cmd/osv/main.go +++ b/tools/osv-linter/cmd/osv/main.go @@ -22,7 +22,7 @@ func main() { Flags: []cli.Flag{ &cli.StringFlag{ Name: "collection", - Value: "osv.dev", + Value: "ALL", Usage: "check collection to use (use 'list' to see)", }, &cli.StringFlag{ diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index e3d69ab7..625e68fe 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -98,17 +98,6 @@ var Collections = []CheckCollectionDef{ CheckPackagePurlValid, }, }, - { - Name: "osv.dev", - Description: "the checks OSV.dev considers necessary for a high quality record", - Checks: []*CheckDef{ - CheckRangeHasIntroducedEvent, - CheckRangeIsDistinct, - CheckPackageExists, - CheckPackageVersionsExist, - CheckPackagePurlValid, - }, - }, } // CollectionFromName returns the CheckCollectionDef with the given name. From 7b5498f5b71066065ec1915853befd4f876f25e5 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:08:15 +0000 Subject: [PATCH 18/32] refactor: eliminate backticks I was following examples in the GJSON documentation, but I can't really see a reason for them. Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/packages.go | 36 +++++++++---------- tools/osv-linter/internal/checks/ranges.go | 20 +++++------ .../osv-linter/internal/helpers/ecosystems.go | 4 +-- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/tools/osv-linter/internal/checks/packages.go b/tools/osv-linter/internal/checks/packages.go index 274b210e..8760d8e9 100644 --- a/tools/osv-linter/internal/checks/packages.go +++ b/tools/osv-linter/internal/checks/packages.go @@ -19,7 +19,7 @@ var CheckPackageExists = &CheckDef{ // PackageExists checks the package exists in the registry for that ecosystem. func PackageExists(json *gjson.Result) (findings []CheckError) { - affectedEntries := json.Get(`affected`) + affectedEntries := json.Get("affected") knownExistent := make(map[string][]string) knownNonexistent := make(map[string][]string) @@ -28,14 +28,14 @@ func PackageExists(json *gjson.Result) (findings []CheckError) { // for ones for packages, on a per-package basis affectedEntries.ForEach(func(key, value gjson.Result) bool { // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. - maybePackage := value.Get(`package`) + maybePackage := value.Get("package") if !maybePackage.Exists() { return true // keep iterating (over affected entries) } // Normalize ecosystems with a colon to their base. // e.g. "Alpine:v3.5" -> "Alpine" - ecosystem := strings.Split(value.Get(`package.ecosystem`).String(), ":")[0] - pkg := value.Get(`package.name`).String() + ecosystem := strings.Split(value.Get("package.ecosystem").String(), ":")[0] + pkg := value.Get("package.name").String() // Avoid unnecessary network traffic for repeat packages. if _, ok := knownExistent[ecosystem]; ok { @@ -80,41 +80,41 @@ var CheckPackageVersionsExist = &CheckDef{ // PackageVersionsExist checks the package versions exist in the registry for that ecosystem. func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { - affectedEntries := json.Get(`affected`) + affectedEntries := json.Get("affected") // Examine each affected entry: // for ones for packages, on a per-package basis affectedEntries.ForEach(func(key, value gjson.Result) bool { // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. - maybePackage := value.Get(`package`) + maybePackage := value.Get("package") if !maybePackage.Exists() { return true // keep iterating (over affected entries) } // Normalize ecosystems with a colon to their base. // e.g. "Alpine:v3.5" -> "Alpine" - ecosystem := strings.Split(value.Get(`package.ecosystem`).String(), ":")[0] - pkg := value.Get(`package.name`).String() + ecosystem := strings.Split(value.Get("package.ecosystem").String(), ":")[0] + pkg := value.Get("package.name").String() versionsToCheck := []string{} // Examine versions in ranges. - maybeRanges := value.Get(`ranges`) + maybeRanges := value.Get("ranges") maybeRanges.ForEach(func(key, value gjson.Result) bool { - rangeType := value.Get(`type`).String() + rangeType := value.Get("type").String() if rangeType == "GIT" { return true // keep iterating (over ranges) } - events := value.Get(`events`) + events := value.Get("events") events.ForEach(func(key, value gjson.Result) bool { // Collect all the introduced values. - result := value.Get(`introduced`) + result := value.Get("introduced") if result.Exists() && result.String() != "0" { versionsToCheck = append(versionsToCheck, result.String()) } // Collect all the fixed/last_affected values. - result = value.Get(`fixed`) + result = value.Get("fixed") if result.Exists() { versionsToCheck = append(versionsToCheck, result.String()) } - result = value.Get(`last_affected`) + result = value.Get("last_affected") if result.Exists() { versionsToCheck = append(versionsToCheck, result.String()) } @@ -123,7 +123,7 @@ func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { return true // keep iterating (over ranges) }) // Examine versions in versions array. - maybeVersions := value.Get(`versions`) + maybeVersions := value.Get("versions") maybeVersions.ForEach(func(key, value gjson.Result) bool { versionsToCheck = append(versionsToCheck, value.String()) return true // keep iterating (over versions) @@ -147,17 +147,17 @@ var CheckPackagePurlValid = &CheckDef{ // PackagePurlValid checks the package purls validate. func PackagePurlValid(json *gjson.Result) (findings []CheckError) { - affectedEntries := json.Get(`affected`) + affectedEntries := json.Get("affected") // Examine each affected entry: // for ones for packages, on a per-package basis affectedEntries.ForEach(func(key, value gjson.Result) bool { // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. - maybePackage := value.Get(`package`) + maybePackage := value.Get("package") if !maybePackage.Exists() { return true // keep iterating (over affected entries) } - purl := value.Get(`package.purl`) + purl := value.Get("package.purl") if !purl.Exists() { return true // keep iterating (over affected entries) } diff --git a/tools/osv-linter/internal/checks/ranges.go b/tools/osv-linter/internal/checks/ranges.go index 22cd50e7..1b88b6b4 100644 --- a/tools/osv-linter/internal/checks/ranges.go +++ b/tools/osv-linter/internal/checks/ranges.go @@ -17,12 +17,12 @@ var CheckRangeHasIntroducedEvent = &CheckDef{ // RangeHasIntroducedEvent checks for missing 'introduced' objects in events. func RangeHasIntroducedEvent(json *gjson.Result) (findings []CheckError) { // It is valid to not have any ranges. - ranges := json.Get(`affected.#(ranges)`) + ranges := json.Get("affected.#(ranges)") if !ranges.Exists() { return nil } - result := json.Get(`affected.#(ranges.#(events.#(introduced)))`) + result := json.Get("affected.#(ranges.#(events.#(introduced)))") if !result.Exists() { findings = append(findings, CheckError{Message: "missing 'introduced' object in event"}) @@ -42,41 +42,41 @@ var CheckRangeIsDistinct = &CheckDef{ // RangeIsDistinct checks that the introduced and fixed (or last_affected) values differ. // (on a per-repo basis for GIT ranges, and on a per-package basis otherwise) func RangeIsDistinct(json *gjson.Result) (findings []CheckError) { - affectedEntries := json.Get(`affected`) + affectedEntries := json.Get("affected") // Examine each entry: // for ones for packages, on a per-package basis // for GIT ranges, on a per-repo basis affectedEntries.ForEach(func(key, value gjson.Result) bool { // If it has a package field, it's for a package, otherwise confirm the range is of type GIT. - maybePackage := value.Get(`package`) - ranges := value.Get(`ranges`) + maybePackage := value.Get("package") + ranges := value.Get("ranges") var pkg bool if maybePackage.Exists() { pkg = true } ranges.ForEach(func(key, value gjson.Result) bool { - rangeType := value.Get(`type`).String() + rangeType := value.Get("type").String() if !pkg && rangeType != "GIT" { findings = append(findings, CheckError{Message: fmt.Sprintf("unexpected range type %q for %s", rangeType, value.String())}) } // Examine the events, collect all of the range starting values and range ending values (fixed and last_affected). // There must be no overlap between these two sets of values. - events := value.Get(`events`) + events := value.Get("events") var startEvents []string var endEvents []string events.ForEach(func(key, value gjson.Result) bool { // Collect all the introduced values. - result := value.Get(`introduced`) + result := value.Get("introduced") if result.Exists() { startEvents = append(startEvents, result.String()) } // Collect all the fixed/last_affected values. - result = value.Get(`fixed`) + result = value.Get("fixed") if result.Exists() { endEvents = append(endEvents, result.String()) } - result = value.Get(`last_affected`) + result = value.Get("last_affected") if result.Exists() { endEvents = append(endEvents, result.String()) } diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index 29f3d146..4320cd5a 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -74,7 +74,7 @@ func PackageVersionsExistInPyPI(pkg string, versions []string) error { } // Fetch all known versions of package. versionsInPyPy := []string{} - releases := gjson.GetBytes(respJSON, `releases.@keys`) + releases := gjson.GetBytes(respJSON, "releases.@keys") releases.ForEach(func(key, value gjson.Result) bool { versionsInPyPy = append(versionsInPyPy, value.String()) return true // keep iterating. @@ -228,7 +228,7 @@ func GoVersionsExist(versions []string) error { } // Fetch all known versions of package. GoVersions := []string{} - releases := gjson.GetBytes(respJSON, `#.version`) + releases := gjson.GetBytes(respJSON, "#.version") releases.ForEach(func(key, value gjson.Result) bool { GoVersions = append(GoVersions, value.String()) return true // keep iterating. From c6999485f48aef1bf488b2e76d4e0b0a1c5e17eb Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:18:10 +0000 Subject: [PATCH 19/32] refactor: address reviewer nit return the result of the comparison instead of branching to return a boolean Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/helpers/ecosystems.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index 4320cd5a..a94c6ad5 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -44,11 +44,8 @@ func PackageExistsInPyPI(pkg string) bool { if err != nil { return false } - if resp.StatusCode == http.StatusOK { - return true - } - return false + return resp.StatusCode == http.StatusOK } // Confirm that all specified versions of a package exist in PyPI. From 4ca72e09e537c474e482f8bfeedf072477d08f0b Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:19:15 +0000 Subject: [PATCH 20/32] build: use Go 1.23.0 New project, no reason not to Signed-off-by: Andrew Pollock --- tools/osv-linter/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index c49907b1..26f4cb8d 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -1,6 +1,6 @@ module github.com/ossf/osv-schema/linter -go 1.21.10 +go 1.23.0 require ( github.com/google/go-cmp v0.6.0 From a6bca417505d21acbfbcd7ac930a695c241d4d8f Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:21:53 +0000 Subject: [PATCH 21/32] build: don't track go.work{,.sum} General wisdom seems to be not to include these in the repo Signed-off-by: Andrew Pollock --- .gitignore | 2 ++ go.work | 3 --- go.work.sum | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 .gitignore delete mode 100644 go.work delete mode 100644 go.work.sum diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..42e9f7c3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +go.work +go.work.sum diff --git a/go.work b/go.work deleted file mode 100644 index 5d2774fa..00000000 --- a/go.work +++ /dev/null @@ -1,3 +0,0 @@ -go 1.21.10 - -use ./tools/osv-linter diff --git a/go.work.sum b/go.work.sum deleted file mode 100644 index 708d15c5..00000000 --- a/go.work.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 4c55775b93b8162f248647d6a6d97e952b5e6d0c Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:32:19 +0000 Subject: [PATCH 22/32] build: use go1.22.6 1.23.0 isn't available for gLinux yet Signed-off-by: Andrew Pollock --- tools/osv-linter/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index 26f4cb8d..e4e67773 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -1,6 +1,6 @@ module github.com/ossf/osv-schema/linter -go 1.23.0 +go 1.22.6 require ( github.com/google/go-cmp v0.6.0 From b6e0cbae308c70f87ce10473b6b6e2cbb5920f80 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:35:13 +0000 Subject: [PATCH 23/32] refactor: eliminate custom string templating for URLs Addresses reviewer feedback Signed-off-by: Andrew Pollock --- .../osv-linter/internal/helpers/ecosystems.go | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/helpers/ecosystems.go index a94c6ad5..e952f5ce 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/helpers/ecosystems.go @@ -35,9 +35,7 @@ func PackageVersionsExistInEcosystem(pkg string, versions []string, ecosystem st // Validate the existence of a package in PyPI. func PackageExistsInPyPI(pkg string) bool { - packageURL := "https://pypi.org/pypi/{package}/json" - - packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkg) // This 404's for non-existent packages. resp, err := Head(packageInstanceURL) @@ -50,9 +48,7 @@ func PackageExistsInPyPI(pkg string) bool { // Confirm that all specified versions of a package exist in PyPI. func PackageVersionsExistInPyPI(pkg string, versions []string) error { - packageURL := "https://pypi.org/pypi/{package}/json" - - packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkg) // This 404's for non-existent packages. resp, err := Get(packageInstanceURL) @@ -93,8 +89,6 @@ func PackageVersionsExistInPyPI(pkg string, versions []string) error { // Validate the existence of a package in Go. func PackageExistsInGo(pkg string) bool { - packageURL := "https://proxy.golang.org/{package}/@v/list" - // Of course the Go runtime exists :-) if pkg == "stdlib" || pkg == "toolchain" { return true @@ -106,18 +100,14 @@ func PackageExistsInGo(pkg string) bool { pkg = strings.ToLower(pkg) } - packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + packageInstanceURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/list", pkg) // This 404's for non-existent packages. resp, err := Head(packageInstanceURL) if err != nil { return false } - if resp.StatusCode == http.StatusOK { - return true - } - - return false + return resp.StatusCode == http.StatusOK } // isGoPseudoVersion checks if a given version string is a Go pseudo-version, @@ -140,8 +130,6 @@ func isGoPseudoVersion(version string) bool { // Confirm that all specified versions of a package exist in Go. func PackageVersionsExistInGo(pkg string, versions []string) error { - packageURL := "https://proxy.golang.org/{package}/@v/list" - if pkg == "stdlib" || pkg == "toolchain" { return GoVersionsExist(versions) } @@ -152,7 +140,7 @@ func PackageVersionsExistInGo(pkg string, versions []string) error { pkg = strings.ToLower(pkg) } - packageInstanceURL := strings.ReplaceAll(packageURL, "{package}", pkg) + packageInstanceURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/list", pkg) // This 404's for non-existent packages. resp, err := Get(packageInstanceURL) From df46529527a59230aac440c81dde5077da2a0b73 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Thu, 15 Aug 2024 04:38:27 +0000 Subject: [PATCH 24/32] refactor: rename CheckCollectionDef to CheckCollection Address reviewer feedback Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/checks.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/osv-linter/internal/checks/checks.go b/tools/osv-linter/internal/checks/checks.go index 625e68fe..bc6ae7fc 100644 --- a/tools/osv-linter/internal/checks/checks.go +++ b/tools/osv-linter/internal/checks/checks.go @@ -50,8 +50,8 @@ func (c *CheckDef) Run(json *gjson.Result) (findings []CheckError) { return findings } -// CheckCollectionDef defines a named collection of checks. -type CheckCollectionDef struct { +// CheckCollection defines a named collection of checks. +type CheckCollection struct { Name string Description string Checks []*CheckDef @@ -77,7 +77,7 @@ func FromName(name string) *CheckDef { return nil } -var Collections = []CheckCollectionDef{ +var Collections = []CheckCollection{ { Name: "ALL", Description: "all checks currently defined", @@ -100,8 +100,8 @@ var Collections = []CheckCollectionDef{ }, } -// CollectionFromName returns the CheckCollectionDef with the given name. -func CollectionFromName(name string) *CheckCollectionDef { +// CollectionFromName returns the CheckCollection with the given name. +func CollectionFromName(name string) *CheckCollection { for _, checkcollection := range Collections { if checkcollection.Name == name { return &checkcollection From 28bfae692d990a76adc362a2d31f0ff9370e5373 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Mon, 19 Aug 2024 00:29:42 +0000 Subject: [PATCH 25/32] refactor: address code review feedback - use better Go package names - break out Go packages more granularly - make some functions private Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/packages.go | 6 ++-- .../utility.go => faulttolerant/http.go} | 5 +-- .../{helpers => pkgchecker}/ecosystems.go | 33 ++++++++++--------- 3 files changed, 23 insertions(+), 21 deletions(-) rename tools/osv-linter/internal/{helpers/utility.go => faulttolerant/http.go} (96%) rename tools/osv-linter/internal/{helpers => pkgchecker}/ecosystems.go (89%) diff --git a/tools/osv-linter/internal/checks/packages.go b/tools/osv-linter/internal/checks/packages.go index 8760d8e9..933a8d38 100644 --- a/tools/osv-linter/internal/checks/packages.go +++ b/tools/osv-linter/internal/checks/packages.go @@ -5,7 +5,7 @@ import ( "slices" "strings" - "github.com/ossf/osv-schema/linter/internal/helpers" + "github.com/ossf/osv-schema/linter/internal/pkgchecker" "github.com/package-url/packageurl-go" "github.com/tidwall/gjson" ) @@ -50,7 +50,7 @@ func PackageExists(json *gjson.Result) (findings []CheckError) { } } // Not cached, determine existence. - if !helpers.PackageExistsInEcosystem(pkg, ecosystem) { + if !pkgchecker.ExistsInEcosystem(pkg, ecosystem) { findings = append(findings, CheckError{Message: fmt.Sprintf("package %q not found", pkg)}) _, ok := knownNonexistent[ecosystem] if ok { @@ -128,7 +128,7 @@ func PackageVersionsExist(json *gjson.Result) (findings []CheckError) { versionsToCheck = append(versionsToCheck, value.String()) return true // keep iterating (over versions) }) - err := helpers.PackageVersionsExistInEcosystem(pkg, versionsToCheck, ecosystem) + err := pkgchecker.VersionsExistInEcosystem(pkg, versionsToCheck, ecosystem) if err != nil { findings = append(findings, CheckError{Message: fmt.Sprintf("Failed to find some versions of %s: %#v", pkg, err)}) } diff --git a/tools/osv-linter/internal/helpers/utility.go b/tools/osv-linter/internal/faulttolerant/http.go similarity index 96% rename from tools/osv-linter/internal/helpers/utility.go rename to tools/osv-linter/internal/faulttolerant/http.go index 7f061398..e9d90147 100644 --- a/tools/osv-linter/internal/helpers/utility.go +++ b/tools/osv-linter/internal/faulttolerant/http.go @@ -1,4 +1,4 @@ -package helpers +package faulttolerant import ( "context" @@ -21,8 +21,9 @@ func Get(url string) (resp *http.Response, err error) { r, err := http.DefaultClient.Do(req) if err != nil { return err + } else { + defer r.Body.Close() } - // defer r.Body.Close() switch r.StatusCode / 100 { case 4: diff --git a/tools/osv-linter/internal/helpers/ecosystems.go b/tools/osv-linter/internal/pkgchecker/ecosystems.go similarity index 89% rename from tools/osv-linter/internal/helpers/ecosystems.go rename to tools/osv-linter/internal/pkgchecker/ecosystems.go index e952f5ce..81dd24ed 100644 --- a/tools/osv-linter/internal/helpers/ecosystems.go +++ b/tools/osv-linter/internal/pkgchecker/ecosystems.go @@ -1,4 +1,4 @@ -package helpers +package pkgchecker import ( "fmt" @@ -8,37 +8,38 @@ import ( "slices" "strings" + "github.com/ossf/osv-schema/linter/internal/faulttolerant" "github.com/tidwall/gjson" ) // Dispatcher for ecosystem-specific package existence checking. -func PackageExistsInEcosystem(pkg string, ecosystem string) bool { +func ExistsInEcosystem(pkg string, ecosystem string) bool { switch ecosystem { case "PyPI": - return PackageExistsInPyPI(pkg) + return existsInPyPI(pkg) case "Go": - return PackageExistsInGo(pkg) + return existsInGo(pkg) } return false } // Dispatcher for ecosystem-specific package version existence checking. -func PackageVersionsExistInEcosystem(pkg string, versions []string, ecosystem string) error { +func VersionsExistInEcosystem(pkg string, versions []string, ecosystem string) error { switch ecosystem { case "PyPI": - return PackageVersionsExistInPyPI(pkg, versions) + return versionsExistInPyPI(pkg, versions) case "Go": - return PackageVersionsExistInGo(pkg, versions) + return versionsExistInGo(pkg, versions) } return fmt.Errorf("unsupported ecosystem: %s", ecosystem) } // Validate the existence of a package in PyPI. -func PackageExistsInPyPI(pkg string) bool { +func existsInPyPI(pkg string) bool { packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkg) // This 404's for non-existent packages. - resp, err := Head(packageInstanceURL) + resp, err := faulttolerant.Head(packageInstanceURL) if err != nil { return false } @@ -47,11 +48,11 @@ func PackageExistsInPyPI(pkg string) bool { } // Confirm that all specified versions of a package exist in PyPI. -func PackageVersionsExistInPyPI(pkg string, versions []string) error { +func versionsExistInPyPI(pkg string, versions []string) error { packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkg) // This 404's for non-existent packages. - resp, err := Get(packageInstanceURL) + resp, err := faulttolerant.Get(packageInstanceURL) if err != nil { return fmt.Errorf("unable to validate package: %v", err) } @@ -88,7 +89,7 @@ func PackageVersionsExistInPyPI(pkg string, versions []string) error { } // Validate the existence of a package in Go. -func PackageExistsInGo(pkg string) bool { +func existsInGo(pkg string) bool { // Of course the Go runtime exists :-) if pkg == "stdlib" || pkg == "toolchain" { return true @@ -103,7 +104,7 @@ func PackageExistsInGo(pkg string) bool { packageInstanceURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/list", pkg) // This 404's for non-existent packages. - resp, err := Head(packageInstanceURL) + resp, err := faulttolerant.Head(packageInstanceURL) if err != nil { return false } @@ -129,7 +130,7 @@ func isGoPseudoVersion(version string) bool { } // Confirm that all specified versions of a package exist in Go. -func PackageVersionsExistInGo(pkg string, versions []string) error { +func versionsExistInGo(pkg string, versions []string) error { if pkg == "stdlib" || pkg == "toolchain" { return GoVersionsExist(versions) } @@ -143,7 +144,7 @@ func PackageVersionsExistInGo(pkg string, versions []string) error { packageInstanceURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/list", pkg) // This 404's for non-existent packages. - resp, err := Get(packageInstanceURL) + resp, err := faulttolerant.Get(packageInstanceURL) if err != nil { return fmt.Errorf("unable to validate package: %v", err) } @@ -196,7 +197,7 @@ func PackageVersionsExistInGo(pkg string, versions []string) error { func GoVersionsExist(versions []string) error { URL := "https://go.dev/dl/?mode=json&include=all" - resp, err := Get(URL) + resp, err := faulttolerant.Get(URL) if err != nil { return fmt.Errorf("unable to validate Go versions: %v", err) } From c9090ae5f595e888b7dbabbe8dc2742c752513e3 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Mon, 19 Aug 2024 01:26:05 +0000 Subject: [PATCH 26/32] refactor: remove erronous defer The callers need to do this as they access the response Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/faulttolerant/http.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/osv-linter/internal/faulttolerant/http.go b/tools/osv-linter/internal/faulttolerant/http.go index e9d90147..0dc908ef 100644 --- a/tools/osv-linter/internal/faulttolerant/http.go +++ b/tools/osv-linter/internal/faulttolerant/http.go @@ -21,8 +21,6 @@ func Get(url string) (resp *http.Response, err error) { r, err := http.DefaultClient.Do(req) if err != nil { return err - } else { - defer r.Body.Close() } switch r.StatusCode / 100 { From 2302267a6a90d4c74234a44a7cd46f562af4b8b7 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Mon, 19 Aug 2024 03:48:41 +0000 Subject: [PATCH 27/32] refactor: simplify existance tracking with a struct map key Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/checks/packages.go | 39 +++++++------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/tools/osv-linter/internal/checks/packages.go b/tools/osv-linter/internal/checks/packages.go index 933a8d38..4fad673b 100644 --- a/tools/osv-linter/internal/checks/packages.go +++ b/tools/osv-linter/internal/checks/packages.go @@ -2,7 +2,6 @@ package checks import ( "fmt" - "slices" "strings" "github.com/ossf/osv-schema/linter/internal/pkgchecker" @@ -17,12 +16,17 @@ var CheckPackageExists = &CheckDef{ Check: PackageExists, } +type Package struct { + Ecosystem string + Name string +} + // PackageExists checks the package exists in the registry for that ecosystem. func PackageExists(json *gjson.Result) (findings []CheckError) { affectedEntries := json.Get("affected") - knownExistent := make(map[string][]string) - knownNonexistent := make(map[string][]string) + knownExistent := make(map[Package]bool) + knownNonexistent := make(map[Package]bool) // Examine each entry: // for ones for packages, on a per-package basis @@ -38,33 +42,18 @@ func PackageExists(json *gjson.Result) (findings []CheckError) { pkg := value.Get("package.name").String() // Avoid unnecessary network traffic for repeat packages. - if _, ok := knownExistent[ecosystem]; ok { - if slices.Contains(knownExistent[ecosystem], pkg) { - return true // keep iterating (over affected entries) - } + if _, ok := knownExistent[Package{Ecosystem: ecosystem, Name: pkg}]; ok { + return true // keep iterating (over affected entries) } - if _, ok := knownNonexistent[ecosystem]; ok { - if slices.Contains(knownNonexistent[ecosystem], pkg) { - // Don't add repeat findings for the same package. - return true // keep iterating (over affected entries) - } + if _, ok := knownNonexistent[Package{Ecosystem: ecosystem, Name: pkg}]; ok { + return true // keep iterating (over affected entries) } // Not cached, determine existence. if !pkgchecker.ExistsInEcosystem(pkg, ecosystem) { - findings = append(findings, CheckError{Message: fmt.Sprintf("package %q not found", pkg)}) - _, ok := knownNonexistent[ecosystem] - if ok { - knownNonexistent[ecosystem] = append(knownNonexistent[ecosystem], pkg) - } else { - knownNonexistent[ecosystem] = []string{pkg} - } + findings = append(findings, CheckError{Message: fmt.Sprintf("package %q not found in %q", pkg, ecosystem)}) + knownNonexistent[Package{Ecosystem: ecosystem, Name: pkg}] = true } else { - _, ok := knownExistent[ecosystem] - if ok { - knownExistent[ecosystem] = append(knownExistent[ecosystem], pkg) - } else { - knownExistent[ecosystem] = []string{pkg} - } + knownExistent[Package{Ecosystem: ecosystem, Name: pkg}] = true } return true // keep iterating (over affected entries) }) From efa3ea05db0987fe5cd0c8d2b54987532e9d748e Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 28 Aug 2024 00:10:35 +0000 Subject: [PATCH 28/32] feat: add all supported ecosystems as fail-open stubs This reduces failure noise for ecosystems yet to be implemented Signed-off-by: Andrew Pollock --- .../internal/pkgchecker/ecosystems.go | 110 +++++++++++++++++- 1 file changed, 105 insertions(+), 5 deletions(-) diff --git a/tools/osv-linter/internal/pkgchecker/ecosystems.go b/tools/osv-linter/internal/pkgchecker/ecosystems.go index 81dd24ed..a5a32691 100644 --- a/tools/osv-linter/internal/pkgchecker/ecosystems.go +++ b/tools/osv-linter/internal/pkgchecker/ecosystems.go @@ -15,10 +15,60 @@ import ( // Dispatcher for ecosystem-specific package existence checking. func ExistsInEcosystem(pkg string, ecosystem string) bool { switch ecosystem { - case "PyPI": - return existsInPyPI(pkg) + case "AlmaLinux": + return true + case "Android": + return true + case "Bitnami": + return true + case "ChainGuard": + return true + case "CRAN": + return true + case "crates.io": + return true + case "Debian": + return true + case "GIT": + return true + case "GitHub Actions": + return true case "Go": return existsInGo(pkg) + case "GSD": + return true + case "Hackage": + return true + case "Hex": + return true + case "Linux": + return true + case "Maven": + return true + case "npm": + return true + case "NuGet": + return true + case "OSS-Fuzz": + return true + case "Packagist": + return true + case "Pub": + return true + case "PyPI": + return existsInPyPI(pkg) + case "Rocky Linux": + return true + case "RubyGems": + return true + case "SwiftURL": + return true + case "Ubuntu": + return true + case "UVI": + return true + case "Wolfi": + return true } return false } @@ -26,10 +76,60 @@ func ExistsInEcosystem(pkg string, ecosystem string) bool { // Dispatcher for ecosystem-specific package version existence checking. func VersionsExistInEcosystem(pkg string, versions []string, ecosystem string) error { switch ecosystem { - case "PyPI": - return versionsExistInPyPI(pkg, versions) + case "AlmaLinux": + return nil + case "Android": + return nil + case "Bitnami": + return nil + case "ChainGuard": + return nil + case "CRAN": + return nil + case "crates.io": + return nil + case "Debian": + return nil + case "GIT": + return nil + case "GitHub Actions": + return nil case "Go": return versionsExistInGo(pkg, versions) + case "GSD": + return nil + case "Hackage": + return nil + case "Hex": + return nil + case "Linux": + return nil + case "Maven": + return nil + case "npm": + return nil + case "NuGet": + return nil + case "OSS-Fuzz": + return nil + case "Packagist": + return nil + case "Pub": + return nil + case "PyPI": + return versionsExistInPyPI(pkg, versions) + case "Rocky Linux": + return nil + case "RubyGems": + return nil + case "SwiftURL": + return nil + case "Ubuntu": + return nil + case "UVI": + return nil + case "Wolfi": + return nil } return fmt.Errorf("unsupported ecosystem: %s", ecosystem) } @@ -82,7 +182,7 @@ func versionsExistInPyPI(pkg string, versions []string) error { versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { - return fmt.Errorf("failed to find %#v for %q", versionsMissing, pkg) + return fmt.Errorf("failed to find %#v for %q in PyPI", versionsMissing, pkg) } return nil From ee45522388d4a1816046e39fffc4f7a1e7991b94 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 28 Aug 2024 00:53:32 +0000 Subject: [PATCH 29/32] feat: support multiple individual explicit checks Addresses user expectation feedback from @another-rex Signed-off-by: Andrew Pollock --- tools/osv-linter/cmd/osv/main.go | 14 +++++++------- tools/osv-linter/internal/linter.go | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tools/osv-linter/cmd/osv/main.go b/tools/osv-linter/cmd/osv/main.go index 21d0538f..95b37b80 100644 --- a/tools/osv-linter/cmd/osv/main.go +++ b/tools/osv-linter/cmd/osv/main.go @@ -18,17 +18,17 @@ func main() { Usage: "operations on OSV records", Subcommands: []*cli.Command{ { - Name: "lint", + Name: "lint", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "collection", + Name: "collection", Value: "ALL", Usage: "check collection to use (use 'list' to see)", }, - &cli.StringFlag{ - Name: "check", - Value: "", - Usage: "explicitly run a single check (use 'list' to see)", + &cli.StringSliceFlag{ + Name: "check", + Value: &cli.StringSlice{}, + Usage: "explicitly run a specific check (use 'list' to see)", }, }, Aliases: []string{"check"}, @@ -43,4 +43,4 @@ func main() { if err := app.Run(os.Args); err != nil { log.Fatal(err) } -} \ No newline at end of file +} diff --git a/tools/osv-linter/internal/linter.go b/tools/osv-linter/internal/linter.go index 13e79bc4..b0ff5861 100644 --- a/tools/osv-linter/internal/linter.go +++ b/tools/osv-linter/internal/linter.go @@ -7,6 +7,7 @@ import ( "log" "os" "path/filepath" + "slices" "github.com/tidwall/gjson" @@ -53,7 +54,7 @@ func LintCommand(cCtx *cli.Context) error { } // List all available checks. - if cCtx.String("check") == "list" { + if slices.Contains(cCtx.StringSlice("check"), "list") { fmt.Printf("Available checks:\n\n") for _, check := range checks.CollectionFromName("ALL").Checks { fmt.Printf("%s: (%s): %s\n", check.Code, check.Name, check.Description) @@ -68,8 +69,18 @@ func LintCommand(cCtx *cli.Context) error { var checksToBeRun []*checks.CheckDef - // Run all the checks in a collection. - if cCtx.String("collection") != "" { + // Run just individual checks. + for _, checkRequested := range cCtx.StringSlice("check") { + // Check the requested check exists. + check := checks.FromCode(checkRequested) + if check == nil { + return fmt.Errorf("%q is not a valid check", checkRequested) + } + checksToBeRun = append(checksToBeRun, check) + } + + // Run all the checks in a collection, if no specific checks requested. + if checksToBeRun == nil && cCtx.String("collection") != "" { fmt.Printf("Running %q check collection on %q\n", cCtx.String("collection"), cCtx.Args()) // Check the requested check collection exists. collection := checks.CollectionFromName(cCtx.String("collection")) @@ -79,16 +90,6 @@ func LintCommand(cCtx *cli.Context) error { checksToBeRun = collection.Checks } - // Run just an individual check, overriding anything discovered from a collection. - if code := cCtx.String("check"); code != "" { - // Check the requested check exists. - check := checks.FromCode(code) - if check == nil { - return fmt.Errorf("%q is not a valid check", code) - } - checksToBeRun = []*checks.CheckDef{check} - } - perFileFindings := map[string][]checks.CheckError{} // Figure out what files to check. From 9fc61f57e516cd3e391c366670b813606d8309c0 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 28 Aug 2024 04:22:38 +0000 Subject: [PATCH 30/32] feat(linter): use standard libraries for Go versions Thanks to @cuixq alerting me to golang.org/x/mod/{module,semver} I can check for psuedoversions and mess with semver versions somewhat more cleanly Signed-off-by: Andrew Pollock --- tools/osv-linter/go.mod | 1 + tools/osv-linter/go.sum | 2 + .../internal/pkgchecker/ecosystems.go | 49 +++++++------------ 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index e4e67773..a1be3f72 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -16,4 +16,5 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + golang.org/x/mod v0.20.0 // indirect ) diff --git a/tools/osv-linter/go.sum b/tools/osv-linter/go.sum index 3cdecc42..f3b45866 100644 --- a/tools/osv-linter/go.sum +++ b/tools/osv-linter/go.sum @@ -18,3 +18,5 @@ github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= diff --git a/tools/osv-linter/internal/pkgchecker/ecosystems.go b/tools/osv-linter/internal/pkgchecker/ecosystems.go index a5a32691..ade51068 100644 --- a/tools/osv-linter/internal/pkgchecker/ecosystems.go +++ b/tools/osv-linter/internal/pkgchecker/ecosystems.go @@ -4,12 +4,13 @@ import ( "fmt" "io" "net/http" - "regexp" "slices" "strings" "github.com/ossf/osv-schema/linter/internal/faulttolerant" "github.com/tidwall/gjson" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) // Dispatcher for ecosystem-specific package existence checking. @@ -136,7 +137,7 @@ func VersionsExistInEcosystem(pkg string, versions []string, ecosystem string) e // Validate the existence of a package in PyPI. func existsInPyPI(pkg string) bool { - packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkg) + packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", strings.ToLower(pkg)) // This 404's for non-existent packages. resp, err := faulttolerant.Head(packageInstanceURL) @@ -149,7 +150,7 @@ func existsInPyPI(pkg string) bool { // Confirm that all specified versions of a package exist in PyPI. func versionsExistInPyPI(pkg string, versions []string) error { - packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkg) + packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", strings.ToLower(pkg)) // This 404's for non-existent packages. resp, err := faulttolerant.Get(packageInstanceURL) @@ -211,28 +212,10 @@ func existsInGo(pkg string) bool { return resp.StatusCode == http.StatusOK } -// isGoPseudoVersion checks if a given version string is a Go pseudo-version, -// including those with pre-release and build metadata segments, -// and handles cases where the pre-release identifier starts with '0.'. -func isGoPseudoVersion(version string) bool { - // Seen in the wild: - // 1.2.0.0 - // 0.5.0-alpha.5.0.20200423152442-f4b650b51dc4 - // 1.0.0-beta - // 1.0.4-0.20180125103619-43913f2f4fbd - // 1.1.10-0.20180427153919-f5cbcbc5cc6f - // 1.16.0-0 - // 2.2.5-rc6.0.20190621200032-0ddffe484adc+incompatible - - // Regular expression to match pseudoversions. - pseudoVersionRegex := regexp.MustCompile(`^(0\.|[0-9]+\.[0-9]+\.)(?:0+|(?:\d+(?:[.-](?:rc)?\d+){0,2})(?:\.(?:0+|(?:\d+(?:[.-]\d+){0,2}))){1,2})([-+].+)?$`) - return pseudoVersionRegex.MatchString(version) -} - // Confirm that all specified versions of a package exist in Go. func versionsExistInGo(pkg string, versions []string) error { if pkg == "stdlib" || pkg == "toolchain" { - return GoVersionsExist(versions) + return goVersionsExist(versions) } // The Go Module Proxy seems to require package names to be lowercase. @@ -273,15 +256,13 @@ func versionsExistInGo(pkg string, versions []string) error { // Determine which referenced versions are missing. versionsMissing := []string{} for _, versionToCheckFor := range versions { - // Add pseudo-version to base version mapping here. // First, detect pseudo-version and skip it. - if isGoPseudoVersion(versionToCheckFor) { + if module.IsPseudoVersion("v" + versionToCheckFor) { // TODO: Try mapping the pseudo-version to a base version and // checking for that instead of skipping. continue } - // Check for both bare versions and "v"-prefixed versions. - if slices.Contains(versionsInGo, versionToCheckFor) || slices.Contains(versionsInGo, "v"+versionToCheckFor) { + if slices.Contains(versionsInGo, semver.Canonical("v"+versionToCheckFor)) { continue } versionsMissing = append(versionsMissing, versionToCheckFor) @@ -294,7 +275,7 @@ func versionsExistInGo(pkg string, versions []string) error { } // Confirm that all specified versions of Go exist. -func GoVersionsExist(versions []string) error { +func goVersionsExist(versions []string) error { URL := "https://go.dev/dl/?mode=json&include=all" resp, err := faulttolerant.Get(URL) @@ -323,13 +304,19 @@ func GoVersionsExist(versions []string) error { // Determine which referenced versions are missing. versionsMissing := []string{} for _, versionToCheckFor := range versions { - if isGoPseudoVersion(versionToCheckFor) { - // TODO: Try mapping the pseudo-version to a base version instead of skipping. - continue - } if slices.Contains(GoVersions, "go"+versionToCheckFor) { continue } + if semver.Prerelease("v"+versionToCheckFor) == "-0" { + // Coerce "1.16.0-0" to "1.16". + if slices.Contains(GoVersions, "go"+strings.TrimPrefix(semver.MajorMinor("v"+versionToCheckFor), "v")) { + continue + } + // Coerce "1.21.0-0" to "1.21.0". + if slices.Contains(GoVersions, "go"+strings.TrimPrefix(strings.TrimSuffix("v"+versionToCheckFor, semver.Prerelease("v"+versionToCheckFor)), "v")) { + continue + } + } versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { From 6e3fd30cd16d6645412b3ef1b53fd67ecaae555d Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 28 Aug 2024 04:25:47 +0000 Subject: [PATCH 31/32] fix(linter): correct variable scoping This variable has no need to be public Signed-off-by: Andrew Pollock --- tools/osv-linter/internal/pkgchecker/ecosystems.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/osv-linter/internal/pkgchecker/ecosystems.go b/tools/osv-linter/internal/pkgchecker/ecosystems.go index ade51068..9a66fa17 100644 --- a/tools/osv-linter/internal/pkgchecker/ecosystems.go +++ b/tools/osv-linter/internal/pkgchecker/ecosystems.go @@ -294,33 +294,33 @@ func goVersionsExist(versions []string) error { return fmt.Errorf("unable to retrieve JSON for Go: %v", err) } // Fetch all known versions of package. - GoVersions := []string{} + goVersions := []string{} releases := gjson.GetBytes(respJSON, "#.version") releases.ForEach(func(key, value gjson.Result) bool { - GoVersions = append(GoVersions, value.String()) + goVersions = append(goVersions, value.String()) return true // keep iterating. }) // Determine which referenced versions are missing. versionsMissing := []string{} for _, versionToCheckFor := range versions { - if slices.Contains(GoVersions, "go"+versionToCheckFor) { + if slices.Contains(goVersions, "go"+versionToCheckFor) { continue } if semver.Prerelease("v"+versionToCheckFor) == "-0" { // Coerce "1.16.0-0" to "1.16". - if slices.Contains(GoVersions, "go"+strings.TrimPrefix(semver.MajorMinor("v"+versionToCheckFor), "v")) { + if slices.Contains(goVersions, "go"+strings.TrimPrefix(semver.MajorMinor("v"+versionToCheckFor), "v")) { continue } // Coerce "1.21.0-0" to "1.21.0". - if slices.Contains(GoVersions, "go"+strings.TrimPrefix(strings.TrimSuffix("v"+versionToCheckFor, semver.Prerelease("v"+versionToCheckFor)), "v")) { + if slices.Contains(goVersions, "go"+strings.TrimPrefix(strings.TrimSuffix("v"+versionToCheckFor, semver.Prerelease("v"+versionToCheckFor)), "v")) { continue } } versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { - return fmt.Errorf("failed to find %+v for Go in %+v", versionsMissing, GoVersions) + return fmt.Errorf("failed to find %+v for Go in %+v", versionsMissing, goVersions) } return nil From a3d5ad97d80d3241f376f22a5e79950f09973a93 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Wed, 28 Aug 2024 09:20:18 +0000 Subject: [PATCH 32/32] feat(linter): handle PyPI better This handles all the idiosyncrasies of PEP440 for versions, and normalizes package names per the documented guidelines. Signed-off-by: Andrew Pollock --- tools/osv-linter/go.mod | 3 ++ tools/osv-linter/go.sum | 18 ++++++++++ .../internal/pkgchecker/ecosystems.go | 36 +++++++++++++++---- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/tools/osv-linter/go.mod b/tools/osv-linter/go.mod index a1be3f72..58de02fd 100644 --- a/tools/osv-linter/go.mod +++ b/tools/osv-linter/go.mod @@ -11,10 +11,13 @@ require ( ) require ( + github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect + github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect golang.org/x/mod v0.20.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/tools/osv-linter/go.sum b/tools/osv-linter/go.sum index f3b45866..9bab3491 100644 --- a/tools/osv-linter/go.sum +++ b/tools/osv-linter/go.sum @@ -1,22 +1,40 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 h1:vmXNl+HDfqqXgr0uY1UgK1GAhps8nbAAtqHNBcgyf+4= +github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46/go.mod h1:olhPNdiiAAMiSujemd1O/sc6GcyePr23f/6uGKtthNg= +github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 h1:rcEG5HI490FF0a7zuvxOxen52ddygCfNVjP0XOCMl+M= +github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492/go.mod h1:9Beu8XsUNNfzml7WBf3QmyPToP1wm1Gj/Vc5UJKqTzU= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs= github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/osv-linter/internal/pkgchecker/ecosystems.go b/tools/osv-linter/internal/pkgchecker/ecosystems.go index 9a66fa17..a41da6db 100644 --- a/tools/osv-linter/internal/pkgchecker/ecosystems.go +++ b/tools/osv-linter/internal/pkgchecker/ecosystems.go @@ -4,13 +4,18 @@ import ( "fmt" "io" "net/http" + "regexp" "slices" "strings" - "github.com/ossf/osv-schema/linter/internal/faulttolerant" - "github.com/tidwall/gjson" "golang.org/x/mod/module" "golang.org/x/mod/semver" + + "github.com/ossf/osv-schema/linter/internal/faulttolerant" + + pep440 "github.com/aquasecurity/go-pep440-version" + + "github.com/tidwall/gjson" ) // Dispatcher for ecosystem-specific package existence checking. @@ -150,7 +155,10 @@ func existsInPyPI(pkg string) bool { // Confirm that all specified versions of a package exist in PyPI. func versionsExistInPyPI(pkg string, versions []string) error { - packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", strings.ToLower(pkg)) + // https://packaging.python.org/en/latest/specifications/name-normalization/ + pythonNormalizationRegex := regexp.MustCompile(`[-_.]+`) + pkgNormalized := strings.ToLower(pythonNormalizationRegex.ReplaceAllString(pkg, "-")) + packageInstanceURL := fmt.Sprintf("https://pypi.org/pypi/%s/json", pkgNormalized) // This 404's for non-existent packages. resp, err := faulttolerant.Get(packageInstanceURL) @@ -177,13 +185,29 @@ func versionsExistInPyPI(pkg string, versions []string) error { // Determine which referenced versions are missing. versionsMissing := []string{} for _, versionToCheckFor := range versions { - if slices.Contains(versionsInPyPy, versionToCheckFor) { + versionFound := false + vc, err := pep440.Parse(versionToCheckFor) + if err != nil { + versionsMissing = append(versionsMissing, versionToCheckFor) + continue + } + for _, pkgversion := range versionsInPyPy { + pv, err := pep440.Parse(pkgversion) + if err != nil { + continue + } + if pv.Equal(vc) { + versionFound = true + break + } + } + if versionFound { continue } versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { - return fmt.Errorf("failed to find %#v for %q in PyPI", versionsMissing, pkg) + return fmt.Errorf("failed to find %#v for %q in PyPI %+v", versionsMissing, pkg, versionsInPyPy) } return nil @@ -268,7 +292,7 @@ func versionsExistInGo(pkg string, versions []string) error { versionsMissing = append(versionsMissing, versionToCheckFor) } if len(versionsMissing) > 0 { - return fmt.Errorf("failed to find %+v for %q in %+v", versionsMissing, pkg, versionsInGo) + return fmt.Errorf("failed to find %+v for %q in Go %+v", versionsMissing, pkg, versionsInGo) } return nil