Skip to content

Commit

Permalink
feat: add h1digest when scanning go.mod (#1405)
Browse files Browse the repository at this point in the history 
Fixes #1277
  • Loading branch information
@kzantow
kzantow authored Dec 20, 2022
1 parent 82f32c7 commit 7b08608
Show file tree
Hide file tree
Showing 15 changed files with 247 additions and 94 deletions.
2 changes: 1 addition & 1 deletion schema/json/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ type artifactMetadataContainer struct {
Python pkg.PythonPackageMetadata
Rpm pkg.RpmMetadata
Cargo pkg.CargoPackageMetadata
Go pkg.GolangBinMetadata
Go pkg.GolangMetadata
Php pkg.PhpComposerJSONMetadata
Dart pkg.DartPubMetadata
Dotnet pkg.DotnetDepsMetadata
Expand Down
6 changes: 3 additions & 3 deletions syft/formats/common/cyclonedxhelpers/component_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,16 +90,16 @@ func Test_encodeComponentProperties(t *testing.T) {
Version: "v0.0.0-20211006190231-62292e806868",
Language: pkg.Go,
Type: pkg.GoModulePkg,
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: "1.17",
Architecture: "amd64",
H1Digest: "h1:KlOXYy8wQWTUJYFgkUI40Lzr06ofg5IRXUK5C7qZt1k=",
},
},
expected: &[]cyclonedx.Property{
{Name: "syft:package:language", Value: pkg.Go.String()},
{Name: "syft:package:metadataType", Value: "GolangBinMetadata"},
{Name: "syft:package:metadataType", Value: "GolangMetadata"},
{Name: "syft:package:type", Value: "go-module"},
{Name: "syft:metadata:architecture", Value: "amd64"},
{Name: "syft:metadata:goCompiledVersion", Value: "1.17"},
Expand Down
2 changes: 1 addition & 1 deletion syft/formats/common/spdxhelpers/to_format_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ func toPackageChecksums(p pkg.Package) ([]common.Checksum, bool) {
})
}
}
case pkg.GolangBinMetadata:
case pkg.GolangMetadata:
// because the H1 digest is found in the Golang metadata we cannot claim that the files were analyzed
algo, hexStr, err := util.HDigestToSHA(meta.H1Digest)
if err != nil {
Expand Down
16 changes: 8 additions & 8 deletions syft/formats/common/spdxhelpers/to_format_model_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ func Test_toPackageChecksums(t *testing.T) {
Name: "test",
Version: "1.0.0",
Language: pkg.Go,
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
H1Digest: "h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=",
},
},
Expand Down Expand Up @@ -381,8 +381,8 @@ func Test_H1Digest(t *testing.T) {
pkg: pkg.Package{
Name: "github.com/googleapis/gnostic",
Version: "v0.5.5",
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
H1Digest: "h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=",
},
},
Expand All @@ -393,8 +393,8 @@ func Test_H1Digest(t *testing.T) {
pkg: pkg.Package{
Name: "github.com/googleapis/gnostic",
Version: "v0.5.5",
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
H1Digest: "h1:9fHAtK0uzzz",
},
},
Expand All @@ -405,8 +405,8 @@ func Test_H1Digest(t *testing.T) {
pkg: pkg.Package{
Name: "github.com/googleapis/gnostic",
Version: "v0.5.5",
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
H1Digest: "h12:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=",
},
},
Expand Down
2 changes: 1 addition & 1 deletion syft/formats/common/spdxhelpers/to_syft_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ func extractMetadata(p *spdx.Package, info pkgInfo) (pkg.MetadataType, interface
h1Digest = digest
break
}
return pkg.GolangBinMetadataType, pkg.GolangBinMetadata{
return pkg.GolangMetadataType, pkg.GolangMetadata{
H1Digest: h1Digest,
}
}
Expand Down
4 changes: 2 additions & 2 deletions syft/formats/common/spdxhelpers/to_syft_model_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,8 @@ func TestH1Digest(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
p := toSyftPackage(&test.pkg)
require.Equal(t, pkg.GolangBinMetadataType, p.MetadataType)
meta := p.Metadata.(pkg.GolangBinMetadata)
require.Equal(t, pkg.GolangMetadataType, p.MetadataType)
meta := p.Metadata.(pkg.GolangMetadata)
require.Equal(t, test.expectedDigest, meta.H1Digest)
})
}
Expand Down
20 changes: 14 additions & 6 deletions syft/formats/syftjson/model/package_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestUnmarshalPackageGolang(t *testing.T) {
"language": "go",
"cpes": [],
"purl": "pkg:golang/gopkg.in/square/go-jose.v2@v2.6.0",
"metadataType": "GolangBinMetadata",
"metadataType": "GolangMetadata",
"metadata": {
"goCompiledVersion": "go1.18",
"architecture": "amd64",
Expand All @@ -43,7 +43,7 @@ func TestUnmarshalPackageGolang(t *testing.T) {
}`),
assert: func(p *Package) {
assert.NotNil(t, p.Metadata)
golangMetadata := p.Metadata.(pkg.GolangBinMetadata)
golangMetadata := p.Metadata.(pkg.GolangMetadata)
assert.NotEmpty(t, golangMetadata)
assert.Equal(t, "go1.18", golangMetadata.GoCompiledVersion)
},
Expand Down Expand Up @@ -93,7 +93,7 @@ func Test_unpackMetadata(t *testing.T) {
}{
{
name: "unmarshal package metadata",
metadataType: pkg.GolangBinMetadataType,
metadataType: pkg.GolangMetadataType,
packageData: []byte(`{
"id": "8b594519bc23da50",
"name": "gopkg.in/square/go-jose.v2",
Expand All @@ -109,7 +109,7 @@ func Test_unpackMetadata(t *testing.T) {
"language": "go",
"cpes": [],
"purl": "pkg:golang/gopkg.in/square/go-jose.v2@v2.6.0",
"metadataType": "GolangBinMetadata",
"metadataType": "GolangMetadata",
"metadata": {
"goCompiledVersion": "go1.18",
"architecture": "amd64",
Expand Down Expand Up @@ -214,11 +214,19 @@ func Test_unpackMetadata(t *testing.T) {
},
{
name: "can handle package with metadata type but missing metadata",
packageData: []byte(`{
"metadataType": "GolangMetadata"
}`),
metadataType: pkg.GolangMetadataType,
wantMetadata: pkg.GolangMetadata{},
},
{
name: "can handle package with golang bin metadata type",
packageData: []byte(`{
"metadataType": "GolangBinMetadata"
}`),
metadataType: pkg.GolangBinMetadataType,
wantMetadata: pkg.GolangBinMetadata{},
metadataType: pkg.GolangMetadataType,
wantMetadata: pkg.GolangMetadata{},
},
{
name: "can handle package with unknonwn metadata type and missing metadata",
Expand Down
4 changes: 2 additions & 2 deletions syft/pkg/cataloger/golang/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ func newGoBinaryPackage(dep *debug.Module, mainModule, goVersion, architecture s
Language: pkg.Go,
Type: pkg.GoModulePkg,
Locations: source.NewLocationSet(locations...),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goVersion,
H1Digest: dep.Sum,
Architecture: architecture,
Expand Down
32 changes: 16 additions & 16 deletions syft/pkg/cataloger/golang/parse_go_binary_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
BuildSettings: defaultBuildSettings,
Expand Down Expand Up @@ -190,8 +190,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{},
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{},
},
},
},
Expand Down Expand Up @@ -233,8 +233,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
H1Digest: "h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic=",
Expand Down Expand Up @@ -285,8 +285,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
BuildSettings: map[string]string{
Expand Down Expand Up @@ -340,8 +340,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
H1Digest: "h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic=",
Expand All @@ -362,8 +362,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
H1Digest: "h1:DYssiUV1pBmKqzKsm4mqXx8artqC0Q8HgZsVI3lMsAg=",
Expand Down Expand Up @@ -417,8 +417,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
H1Digest: "h1:PjhxBct4MZii8FFR8+oeS7QOvxKOTZXgk63EU2XpfJE=",
Expand All @@ -438,8 +438,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
},
},
),
MetadataType: pkg.GolangBinMetadataType,
Metadata: pkg.GolangBinMetadata{
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
GoCompiledVersion: goCompiledVersion,
Architecture: archDetails,
H1Digest: "h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI=",
Expand Down
79 changes: 66 additions & 13 deletions syft/pkg/cataloger/golang/parse_go_mod.go
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
package golang

import (
"bufio"
"fmt"
"io"
"sort"
"strings"

"golang.org/x/mod/modfile"

"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)

// parseGoModFile takes a go.mod and lists all packages discovered.
func parseGoModFile(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
func parseGoModFile(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
packages := make(map[string]pkg.Package)

contents, err := io.ReadAll(reader)
Expand All @@ -27,26 +30,39 @@ func parseGoModFile(_ source.FileResolver, _ *generic.Environment, reader source
return nil, nil, fmt.Errorf("failed to parse go module: %w", err)
}

digests, err := parseGoSumFile(resolver, reader)
if err != nil {
log.Debugf("unable to get go.sum: %v", err)
}

for _, m := range file.Require {
packages[m.Mod.Path] = pkg.Package{
Name: m.Mod.Path,
Version: m.Mod.Version,
Locations: source.NewLocationSet(reader.Location),
PURL: packageURL(m.Mod.Path, m.Mod.Version),
Language: pkg.Go,
Type: pkg.GoModulePkg,
Name: m.Mod.Path,
Version: m.Mod.Version,
Locations: source.NewLocationSet(reader.Location),
PURL: packageURL(m.Mod.Path, m.Mod.Version),
Language: pkg.Go,
Type: pkg.GoModulePkg,
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
H1Digest: digests[fmt.Sprintf("%s %s", m.Mod.Path, m.Mod.Version)],
},
}
}

// remove any old packages and replace with new ones...
for _, m := range file.Replace {
packages[m.New.Path] = pkg.Package{
Name: m.New.Path,
Version: m.New.Version,
Locations: source.NewLocationSet(reader.Location),
PURL: packageURL(m.New.Path, m.New.Version),
Language: pkg.Go,
Type: pkg.GoModulePkg,
Name: m.New.Path,
Version: m.New.Version,
Locations: source.NewLocationSet(reader.Location),
PURL: packageURL(m.New.Path, m.New.Version),
Language: pkg.Go,
Type: pkg.GoModulePkg,
MetadataType: pkg.GolangMetadataType,
Metadata: pkg.GolangMetadata{
H1Digest: digests[fmt.Sprintf("%s %s", m.New.Path, m.New.Version)],
},
}
}

Expand All @@ -69,3 +85,40 @@ func parseGoModFile(_ source.FileResolver, _ *generic.Environment, reader source

return pkgsSlice, nil, nil
}

func parseGoSumFile(resolver source.FileResolver, reader source.LocationReadCloser) (map[string]string, error) {
out := map[string]string{}

if resolver == nil {
return out, fmt.Errorf("no resolver provided")
}

goSumPath := strings.TrimSuffix(reader.Location.RealPath, ".mod") + ".sum"
goSumLocation := resolver.RelativeFileByPath(reader.Location, goSumPath)
if goSumLocation == nil {
return nil, fmt.Errorf("unable to resolve: %s", goSumPath)
}
contents, err := resolver.FileContentsByLocation(*goSumLocation)
if err != nil {
return nil, err
}

// go.sum has the format like:
// github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
// github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
// github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
scanner := bufio.NewScanner(contents)
// optionally, resize scanner's capacity for lines over 64K, see next example
for scanner.Scan() {
line := scanner.Text()
parts := strings.Split(line, " ")
if len(parts) < 3 {
continue
}
nameVersion := fmt.Sprintf("%s %s", parts[0], parts[1])
hash := parts[2]
out[nameVersion] = hash
}

return out, nil
}
Loading

0 comments on commit 7b08608

Please sign in to comment.