Initial commit of OSV record linter

.gitignore

@@ -0,0 +1,2 @@
+go.work
+go.work.sum

tools/osv-linter/cmd/osv/main.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/ossf/osv-schema/linter/internal"
+	"github.com/urfave/cli/v2"
+)
+
+func main() {
+	app := &cli.App{
+		Name:  "osv",
+		Usage: "OSV general purpose tool",
+		Commands: []*cli.Command{
+			{
+				Name:  "record",
+				Usage: "operations on OSV records",
+				Subcommands: []*cli.Command{
+					{
+						Name: "lint",
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:  "collection",
+								Value: "ALL",
+								Usage: "check collection to use (use 'list' to see)",
+							},
+							&cli.StringSliceFlag{
+								Name:  "check",
+								Value: &cli.StringSlice{},
+								Usage: "explicitly run a specific check (use 'list' to see)",
+							},
+						},
+						Aliases: []string{"check"},
+						Usage:   "check OSV records for correctness",
+						Action:  internal.LintCommand,
+					},
+				},
+			},
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}

tools/osv-linter/go.mod

@@ -0,0 +1,23 @@
+module github.com/ossf/osv-schema/linter
+
+go 1.22.6
+
+require (
+	github.com/google/go-cmp v0.6.0
+	github.com/package-url/packageurl-go v0.1.3
+	github.com/sethvargo/go-retry v0.2.4
+	github.com/tidwall/gjson v1.17.1
+	github.com/urfave/cli/v2 v2.27.2
+)
+
+require (
+	github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect
+	github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+)

tools/osv-linter/go.sum

@@ -0,0 +1,40 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 h1:vmXNl+HDfqqXgr0uY1UgK1GAhps8nbAAtqHNBcgyf+4=
+github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46/go.mod h1:olhPNdiiAAMiSujemd1O/sc6GcyePr23f/6uGKtthNg=
+github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 h1:rcEG5HI490FF0a7zuvxOxen52ddygCfNVjP0XOCMl+M=
+github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492/go.mod h1:9Beu8XsUNNfzml7WBf3QmyPToP1wm1Gj/Vc5UJKqTzU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs=
+github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
+github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
+github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/osv-linter/internal/checks/checks.go

@@ -0,0 +1,111 @@
+// Package checks defines and implements all checks and collections of checks.
+//
+// To add additional checks:
+// 1. define a new instance of `Check`
+// 2. add it to the `checks` array
+// 3. add it to the relevent collections defined in `checkCollections`
+//
+// To add additional collections of checks:
+// 1. add to the `checkCollections` array.
+package checks
+
+import (
+	"fmt"
+
+	"github.com/tidwall/gjson"
+)
+
+// CheckError describes when a check fails.
+type CheckError struct {
+	Code    string
+	Message string
+}
+
+// Error returns the error message, including the code.
+func (ce *CheckError) Error() string {
+	return fmt.Sprintf("%s: %s", ce.Code, ce.Message)
+}
+
+// CheckDef defines a single check.
+type CheckDef struct {
+	Code        string
+	Name        string
+	Description string
+	Check       Check
+}
+
+// Check defines how to run the check.
+type Check func(*gjson.Result) []CheckError
+
+// Run runs the check, returning any findings.
+// The check has no awareness of the check's Code,
+// this merges that with the check's findings.
+func (c *CheckDef) Run(json *gjson.Result) (findings []CheckError) {
+	for _, finding := range c.Check(json) {
+		findings = append(findings, CheckError{
+			Code:    c.Code,
+			Message: finding.Error(),
+		})
+	}
+	return findings
+}
+
+// CheckCollection defines a named collection of checks.
+type CheckCollection struct {
+	Name        string
+	Description string
+	Checks      []*CheckDef
+}
+
+// FromCode returns the check with a specific code.
+func FromCode(code string) *CheckDef {
+	for _, check := range CollectionFromName("ALL").Checks {
+		if check.Code == code {
+			return check
+		}
+	}
+	return nil
+}
+
+// FromName returns the check with a specific name.
+func FromName(name string) *CheckDef {
+	for _, check := range CollectionFromName("ALL").Checks {
+		if check.Name == name {
+			return check
+		}
+	}
+	return nil
+}
+
+var Collections = []CheckCollection{
+	{
+		Name:        "ALL",
+		Description: "all checks currently defined",
+		Checks: []*CheckDef{
+			CheckRangeHasIntroducedEvent,
+			CheckRangeIsDistinct,
+			CheckPackageExists,
+			CheckPackageVersionsExist,
+			CheckPackagePurlValid,
+		},
+	},
+	{
+		Name:        "offline",
+		Description: "checks that do not have remote data dependencies",
+		Checks: []*CheckDef{
+			CheckRangeHasIntroducedEvent,
+			CheckRangeIsDistinct,
+			CheckPackagePurlValid,
+		},
+	},
+}
+
+// CollectionFromName returns the CheckCollection with the given name.
+func CollectionFromName(name string) *CheckCollection {
+	for _, checkcollection := range Collections {
+		if checkcollection.Name == name {
+			return &checkcollection
+		}
+	}
+	return nil
+}

tools/osv-linter/internal/checks/packages.go

@@ -0,0 +1,162 @@
+package checks
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ossf/osv-schema/linter/internal/pkgchecker"
+	"github.com/package-url/packageurl-go"
+	"github.com/tidwall/gjson"
+)
+
+var CheckPackageExists = &CheckDef{
+	Code:        "P0001",
+	Name:        "package-exists",
+	Description: "package exists in ecosystem's registry",
+	Check:       PackageExists,
+}
+
+type Package struct {
+	Ecosystem string
+	Name      string
+}
+
+// PackageExists checks the package exists in the registry for that ecosystem.
+func PackageExists(json *gjson.Result) (findings []CheckError) {
+	affectedEntries := json.Get("affected")
+
+	knownExistent := make(map[Package]bool)
+	knownNonexistent := make(map[Package]bool)
+
+	// Examine each entry:
+	// for ones for packages, on a per-package basis
+	affectedEntries.ForEach(func(key, value gjson.Result) bool {
+		// If it has a package field, it's for a package, otherwise confirm the range is of type GIT.
+		maybePackage := value.Get("package")
+		if !maybePackage.Exists() {
+			return true // keep iterating (over affected entries)
+		}
+		// Normalize ecosystems with a colon to their base.
+		// e.g. "Alpine:v3.5" -> "Alpine"
+		ecosystem := strings.Split(value.Get("package.ecosystem").String(), ":")[0]
+		pkg := value.Get("package.name").String()
+
+		// Avoid unnecessary network traffic for repeat packages.
+		if _, ok := knownExistent[Package{Ecosystem: ecosystem, Name: pkg}]; ok {
+			return true // keep iterating (over affected entries)
+		}
+		if _, ok := knownNonexistent[Package{Ecosystem: ecosystem, Name: pkg}]; ok {
+			return true // keep iterating (over affected entries)
+		}
+		// Not cached, determine existence.
+		if !pkgchecker.ExistsInEcosystem(pkg, ecosystem) {
+			findings = append(findings, CheckError{Message: fmt.Sprintf("package %q not found in %q", pkg, ecosystem)})
+			knownNonexistent[Package{Ecosystem: ecosystem, Name: pkg}] = true
+		} else {
+			knownExistent[Package{Ecosystem: ecosystem, Name: pkg}] = true
+		}
+		return true // keep iterating (over affected entries)
+	})
+	return findings
+}
+
+var CheckPackageVersionsExist = &CheckDef{
+	Code:        "P0002",
+	Name:        "package-versions-exist",
+	Description: "package versions exist in ecosystem's registry",
+	Check:       PackageVersionsExist,
+}
+
+// PackageVersionsExist checks the package versions exist in the registry for that ecosystem.
+func PackageVersionsExist(json *gjson.Result) (findings []CheckError) {
+	affectedEntries := json.Get("affected")
+
+	// Examine each affected entry:
+	// for ones for packages, on a per-package basis
+	affectedEntries.ForEach(func(key, value gjson.Result) bool {
+		// If it has a package field, it's for a package, otherwise confirm the range is of type GIT.
+		maybePackage := value.Get("package")
+		if !maybePackage.Exists() {
+			return true // keep iterating (over affected entries)
+		}
+		// Normalize ecosystems with a colon to their base.
+		// e.g. "Alpine:v3.5" -> "Alpine"
+		ecosystem := strings.Split(value.Get("package.ecosystem").String(), ":")[0]
+		pkg := value.Get("package.name").String()
+		versionsToCheck := []string{}
+		// Examine versions in ranges.
+		maybeRanges := value.Get("ranges")
+		maybeRanges.ForEach(func(key, value gjson.Result) bool {
+			rangeType := value.Get("type").String()
+			if rangeType == "GIT" {
+				return true // keep iterating (over ranges)
+			}
+			events := value.Get("events")
+			events.ForEach(func(key, value gjson.Result) bool {
+				// Collect all the introduced values.
+				result := value.Get("introduced")
+				if result.Exists() && result.String() != "0" {
+					versionsToCheck = append(versionsToCheck, result.String())
+				}
+				// Collect all the fixed/last_affected values.
+				result = value.Get("fixed")
+				if result.Exists() {
+					versionsToCheck = append(versionsToCheck, result.String())
+				}
+				result = value.Get("last_affected")
+				if result.Exists() {
+					versionsToCheck = append(versionsToCheck, result.String())
+				}
+				return true // keep iterating (over events)
+			})
+			return true // keep iterating (over ranges)
+		})
+		// Examine versions in versions array.
+		maybeVersions := value.Get("versions")
+		maybeVersions.ForEach(func(key, value gjson.Result) bool {
+			versionsToCheck = append(versionsToCheck, value.String())
+			return true // keep iterating (over versions)
+		})
+		err := pkgchecker.VersionsExistInEcosystem(pkg, versionsToCheck, ecosystem)
+		if err != nil {
+			findings = append(findings, CheckError{Message: fmt.Sprintf("Failed to find some versions of %s: %#v", pkg, err)})
+		}
+
+		return true // keep iterating (over affected entries)
+	})
+	return findings
+}
+
+var CheckPackagePurlValid = &CheckDef{
+	Code:        "P0003",
+	Name:        "package-purl-valid",
+	Description: "package purl validates",
+	Check:       PackagePurlValid,
+}
+
+// PackagePurlValid checks the package purls validate.
+func PackagePurlValid(json *gjson.Result) (findings []CheckError) {
+	affectedEntries := json.Get("affected")
+
+	// Examine each affected entry:
+	// for ones for packages, on a per-package basis
+	affectedEntries.ForEach(func(key, value gjson.Result) bool {
+		// If it has a package field, it's for a package, otherwise confirm the range is of type GIT.
+		maybePackage := value.Get("package")
+		if !maybePackage.Exists() {
+			return true // keep iterating (over affected entries)
+		}
+		purl := value.Get("package.purl")
+		if !purl.Exists() {
+			return true // keep iterating (over affected entries)
+		}
+
+		_, err := packageurl.FromString(purl.String())
+		if err != nil {
+			findings = append(findings, CheckError{Message: fmt.Sprintf("Invalid Purl %q: %#v", purl.String(), err)})
+		}
+
+		return true // keep iterating (over affected entries)
+	})
+	return findings
+}