feat: scan local go mod cache for licenses of golang packages (#1645)

Signed-off-by: Avi Deitcher <avi@deitcher.net>
Co-authored-by: Keith Zantow <kzantow@gmail.com>

README.md

@@ -494,6 +494,16 @@ package:
     # same as -s ; SYFT_PACKAGE_CATALOGER_SCOPE env var
     scope: "squashed"
 
+golang:
+   # search for go package licences in the GOPATH of the system running Syft, note that this is outside the
+   # container filesystem and potentially outside the root of a local directory scan
+   # SYFT_GOLANG_SEARCH_LOCAL_MOD_CACHE_LICENSES env var
+   search-local-mod-cache-licenses: false
+
+   # specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod
+   # SYFT_GOLANG_LOCAL_MOD_CACHE_DIR env var
+   local-mod-cache-dir: ""
+
 # cataloging file contents is exposed through the power-user subcommand
 file-contents:
   cataloger:

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/anchore/stereoscope v0.0.0-20230317134707-7928713c391e
 	github.com/docker/docker v23.0.1+incompatible
 	github.com/google/go-containerregistry v0.14.0
+	github.com/google/licensecheck v0.3.1
 	github.com/invopop/jsonschema v0.7.0
 	github.com/knqyf263/go-rpmdb v0.0.0-20221030135625-4082a22221ce
 	github.com/opencontainers/go-digest v1.0.0

go.sum

@@ -264,6 +264,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
 github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs=
+github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

internal/config/application.go

@@ -18,6 +18,7 @@ import (
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg/cataloger"
+	golangCataloger "github.com/anchore/syft/syft/pkg/cataloger/golang"
 )
 
 var (
@@ -48,6 +49,7 @@ type Application struct {
 	Log                logging            `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options
 	Catalogers         []string           `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"`
 	Package            pkg                `yaml:"package" json:"package" mapstructure:"package"`
+	Golang             golang             `yaml:"golang" json:"golang" mapstructure:"golang"`
 	Attest             attest             `yaml:"attest" json:"attest" mapstructure:"attest"`
 	FileMetadata       FileMetadata       `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
 	FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
@@ -69,6 +71,10 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
 		},
 		Catalogers:  cfg.Catalogers,
 		Parallelism: cfg.Parallelism,
+		Golang: golangCataloger.GoCatalogerOpts{
+			SearchLocalModCacheLicenses: cfg.Golang.SearchLocalModCacheLicenses,
+			LocalModCacheDir:            cfg.Golang.LocalModCacheDir,
+		},
 	}
 }
 

internal/config/golang.go

@@ -0,0 +1,13 @@
+package config
+
+import "github.com/spf13/viper"
+
+type golang struct {
+	SearchLocalModCacheLicenses bool   `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
+	LocalModCacheDir            string `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
+}
+
+func (cfg golang) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("golang.search-local-mod-cache-licenses", false)
+	v.SetDefault("golang.local-mod-cache-dir", "")
+}

internal/licenses/list.go

@@ -0,0 +1,53 @@
+package licenses
+
+import "github.com/anchore/syft/internal"
+
+// all of these taken from https://github.com/golang/pkgsite/blob/8996ff632abee854aef1b764ca0501f262f8f523/internal/licenses/licenses.go#L338
+// which unfortunately is not exported. But fortunately is under BSD-style license.
+
+var (
+	FileNames = []string{
+		"COPYING",
+		"COPYING.md",
+		"COPYING.markdown",
+		"COPYING.txt",
+		"LICENCE",
+		"LICENCE.md",
+		"LICENCE.markdown",
+		"LICENCE.txt",
+		"LICENSE",
+		"LICENSE.md",
+		"LICENSE.markdown",
+		"LICENSE.txt",
+		"LICENSE-2.0.txt",
+		"LICENCE-2.0.txt",
+		"LICENSE-APACHE",
+		"LICENCE-APACHE",
+		"LICENSE-APACHE-2.0.txt",
+		"LICENCE-APACHE-2.0.txt",
+		"LICENSE-MIT",
+		"LICENCE-MIT",
+		"LICENSE.MIT",
+		"LICENCE.MIT",
+		"LICENSE.code",
+		"LICENCE.code",
+		"LICENSE.docs",
+		"LICENCE.docs",
+		"LICENSE.rst",
+		"LICENCE.rst",
+		"MIT-LICENSE",
+		"MIT-LICENCE",
+		"MIT-LICENSE.md",
+		"MIT-LICENCE.md",
+		"MIT-LICENSE.markdown",
+		"MIT-LICENCE.markdown",
+		"MIT-LICENSE.txt",
+		"MIT-LICENCE.txt",
+		"MIT_LICENSE",
+		"MIT_LICENCE",
+		"UNLICENSE",
+		"UNLICENCE",
+	}
+
+	FileNameSet = internal.NewStringSet(FileNames...)
+)

internal/licenses/parser.go

@@ -0,0 +1,33 @@
+package licenses
+
+import (
+	"io"
+
+	"github.com/google/licensecheck"
+	"golang.org/x/exp/slices"
+)
+
+const (
+	coverageThreshold  = 75
+	unknownLicenseType = "UNKNOWN"
+)
+
+// Parse scans the contents of a license file to attempt to determine the type of license it is
+func Parse(reader io.Reader) (licenses []string, err error) {
+	contents, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	cov := licensecheck.Scan(contents)
+
+	if cov.Percent < float64(coverageThreshold) {
+		licenses = append(licenses, unknownLicenseType)
+	}
+	for _, m := range cov.Match {
+		if slices.Contains(licenses, m.ID) {
+			continue
+		}
+		licenses = append(licenses, m.ID)
+	}
+	return
+}

syft/pkg/cataloger/cataloger.go

@@ -48,7 +48,7 @@ func ImageCatalogers(cfg Config) []pkg.Cataloger {
 		java.NewJavaCataloger(cfg.Java()),
 		java.NewNativeImageCataloger(),
 		apkdb.NewApkdbCataloger(),
-		golang.NewGoModuleBinaryCataloger(),
+		golang.NewGoModuleBinaryCataloger(cfg.Go()),
 		dotnet.NewDotnetDepsCataloger(),
 		portage.NewPortageCataloger(),
 		sbom.NewSBOMCataloger(),
@@ -72,8 +72,8 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
 		java.NewJavaPomCataloger(),
 		java.NewNativeImageCataloger(),
 		apkdb.NewApkdbCataloger(),
-		golang.NewGoModuleBinaryCataloger(),
-		golang.NewGoModFileCataloger(),
+		golang.NewGoModuleBinaryCataloger(cfg.Go()),
+		golang.NewGoModFileCataloger(cfg.Go()),
 		rust.NewCargoLockCataloger(),
 		dart.NewPubspecLockCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
@@ -105,8 +105,8 @@ func AllCatalogers(cfg Config) []pkg.Cataloger {
 		java.NewJavaPomCataloger(),
 		java.NewNativeImageCataloger(),
 		apkdb.NewApkdbCataloger(),
-		golang.NewGoModuleBinaryCataloger(),
-		golang.NewGoModFileCataloger(),
+		golang.NewGoModuleBinaryCataloger(cfg.Go()),
+		golang.NewGoModFileCataloger(cfg.Go()),
 		rust.NewCargoLockCataloger(),
 		rust.NewAuditBinaryCataloger(),
 		dart.NewPubspecLockCataloger(),

syft/pkg/cataloger/config.go

@@ -1,11 +1,13 @@
 package cataloger
 
 import (
+	"github.com/anchore/syft/syft/pkg/cataloger/golang"
 	"github.com/anchore/syft/syft/pkg/cataloger/java"
 )
 
 type Config struct {
 	Search      SearchConfig
+	Golang      golang.GoCatalogerOpts
 	Catalogers  []string
 	Parallelism int
 }
@@ -23,3 +25,7 @@ func (c Config) Java() java.Config {
 		SearchIndexedArchives:   c.Search.IncludeIndexedArchives,
 	}
 }
+
+func (c Config) Go() golang.GoCatalogerOpts {
+	return c.Golang
+}

syft/pkg/cataloger/golang/cataloger.go

@@ -8,14 +8,25 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
+type GoCatalogerOpts struct {
+	SearchLocalModCacheLicenses bool
+	LocalModCacheDir            string
+}
+
 // NewGoModFileCataloger returns a new Go module cataloger object.
-func NewGoModFileCataloger() *generic.Cataloger {
+func NewGoModFileCataloger(opts GoCatalogerOpts) *generic.Cataloger {
+	c := goModCataloger{
+		licenses: newGoLicenses(opts),
+	}
 	return generic.NewCataloger("go-mod-file-cataloger").
-		WithParserByGlobs(parseGoModFile, "**/go.mod")
+		WithParserByGlobs(c.parseGoModFile, "**/go.mod")
 }
 
 // NewGoModuleBinaryCataloger returns a new Golang cataloger object.
-func NewGoModuleBinaryCataloger() *generic.Cataloger {
+func NewGoModuleBinaryCataloger(opts GoCatalogerOpts) *generic.Cataloger {
+	c := goBinaryCataloger{
+		licenses: newGoLicenses(opts),
+	}
 	return generic.NewCataloger("go-module-binary-cataloger").
-		WithParserByMimeTypes(parseGoBinary, internal.ExecutableMIMETypeSet.List()...)
+		WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...)
 }

syft/pkg/cataloger/golang/cataloger_test.go

@@ -27,7 +27,7 @@ func Test_Mod_Cataloger_Globs(t *testing.T) {
 				FromDirectory(t, test.fixture).
 				ExpectsResolverContentQueries(test.expected).
 				IgnoreUnfulfilledPathResponses("src/go.sum").
-				TestCataloger(t, NewGoModFileCataloger())
+				TestCataloger(t, NewGoModFileCataloger(GoCatalogerOpts{}))
 		})
 	}
 }
@@ -52,7 +52,7 @@ func Test_Binary_Cataloger_Globs(t *testing.T) {
 			pkgtest.NewCatalogTester().
 				FromDirectory(t, test.fixture).
 				ExpectsResolverContentQueries(test.expected).
-				TestCataloger(t, NewGoModuleBinaryCataloger())
+				TestCataloger(t, NewGoModuleBinaryCataloger(GoCatalogerOpts{}))
 		})
 	}
 }