Skip to content

Commit

Permalink
Merge branch 'main' into 1577-license-revamp
Browse files Browse the repository at this point in the history
* main:
  chore(deps): bump modernc.org/sqlite from 1.21.2 to 1.22.0 (#1758)
  chore: go-rpmdb update (#1757)
  chore(deps): bump github.com/CycloneDX/cyclonedx-go from 0.7.1-0.20221222100750-41a1ac565cce to 0.7.1 (#1706)
  fix: Improve pnpm support (#1752)
  feat: Add template func `hasField` (#1754)
  fix: only cache java packages and not source content (#1750)

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>
  • Loading branch information
spiffcs committed Apr 26, 2023
2 parents edb7f1c + 02bd527 commit 67def34
Show file tree
Hide file tree
Showing 20 changed files with 297 additions and 76 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/validations.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
- name: Restore Java test-fixture cache
uses: actions/cache@v3
with:
path: syft/pkg/cataloger/java/test-fixtures/java-builds
path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages
key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint' ) }}

- name: Restore RPM test-fixture cache
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,8 @@ Which would produce output like:

Syft also includes a vast array of utility templating functions from [sprig](http://masterminds.github.io/sprig/) apart from the default Golang [text/template](https://pkg.go.dev/text/template#hdr-Functions) to allow users to customize the output format.

Lastly, Syft has custom templating functions defined in `./syft/format/template/encoder.go` to help parse the passed-in JSON structs.

## Multiple outputs

Syft can also output _multiple_ files in differing formats by appending
Expand Down
2 changes: 2 additions & 0 deletions cmd/syft/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package main
import (
"log"

_ "modernc.org/sqlite"

"github.com/anchore/syft/cmd/syft/cli"
)

Expand Down
12 changes: 11 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ require (
github.com/vbatts/go-mtree v0.5.3
golang.org/x/exp v0.0.0-20230202163644-54bba9f4231b
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.22.0
)

require (
Expand Down Expand Up @@ -103,6 +104,7 @@ require (
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.16.4 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
Expand All @@ -120,6 +122,7 @@ require (
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
Expand Down Expand Up @@ -148,8 +151,15 @@ require (
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.22.4 // indirect
modernc.org/sqlite v1.22.0 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect
)

require (
Expand Down
24 changes: 24 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
Expand Down Expand Up @@ -391,6 +392,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
Expand Down Expand Up @@ -444,6 +447,7 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
Expand Down Expand Up @@ -525,7 +529,9 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
Expand Down Expand Up @@ -1170,12 +1176,30 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/libc v1.22.4 h1:wymSbZb0AlrjdAVX3cjreCHTPCpPARbQXNz6BHPzdwQ=
modernc.org/libc v1.22.4/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.22.0 h1:Uo+wEWePCspy4SAu0w2VbzUHEftOs7yoaWX/cYjsq84=
modernc.org/sqlite v1.22.0/go.mod h1:cxbLkB5WS32DnQqeH4h4o1B0eMr8W/y8/RGuxQ3JsC0=
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/tcl v1.15.1 h1:mOQwiEK4p7HruMZcwKTZPw/aqtGM4aY00uzWhlKKYws=
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
Expand Down
8 changes: 8 additions & 0 deletions schema/json/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,17 @@ func build() *jsonschema.Schema {
}
documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftjsonModel.Document{}))
metadataSchema := reflector.ReflectFromType(reflect.TypeOf(&artifactMetadataContainer{}))
licenseSchema := reflector.ReflectFromType(reflect.TypeOf(&[]pkg.License{}))

// TODO: inject source definitions

// inject the definitions of licenses into the schema definitions
// we do this since the license tag is "-" in the syftjson model
// - is used for previous versions of syftjson where licenses could be []string
// because we migrated to a more complex struct we have to manually set this in the generate code
// in order to maintain backwards compatibility
documentSchema.Definitions["Package"].Properties.Set("licenses", licenseSchema)

// inject the definitions of all metadatas into the schema definitions

var metadataNames []string
Expand Down
90 changes: 57 additions & 33 deletions schema/json/schema-8.0.0.json
Original file line number Diff line number Diff line change
Expand Up @@ -737,32 +737,6 @@
"kb"
]
},
"License": {
"properties": {
"value": {
"type": "string"
},
"spdx-expression": {
"type": "string"
},
"type": {
"type": "string"
},
"url": {
"type": "string"
},
"location": {
"$ref": "#/$defs/Location"
}
},
"type": "object",
"required": [
"value",
"spdx-expression",
"type",
"url"
]
},
"LinuxKernelMetadata": {
"properties": {
"name": {
Expand Down Expand Up @@ -1055,12 +1029,6 @@
},
"type": "array"
},
"licenses": {
"items": {
"$ref": "#/$defs/License"
},
"type": "array"
},
"language": {
"type": "string"
},
Expand Down Expand Up @@ -1166,6 +1134,63 @@
"$ref": "#/$defs/RpmMetadata"
}
]
},
"licenses": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$defs": {
"License": {
"properties": {
"value": {
"type": "string"
},
"spdxExpression": {
"type": "string"
},
"type": {
"type": "string"
},
"url": {
"type": "string"
},
"location": {
"$ref": "#/$defs/Location"
}
},
"type": "object",
"required": [
"value",
"spdxExpression",
"type",
"url"
]
},
"Location": {
"properties": {
"path": {
"type": "string"
},
"layerID": {
"type": "string"
},
"annotations": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object"
}
},
"type": "object",
"required": [
"path"
]
}
},
"items": {
"$ref": "#/$defs/License"
},
"type": "array"
}
},
"type": "object",
Expand All @@ -1176,7 +1201,6 @@
"type",
"foundBy",
"locations",
"licenses",
"language",
"cpes",
"purl"
Expand Down
20 changes: 12 additions & 8 deletions syft/formats/cyclonedxjson/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,18 @@ func TestCycloneDxImageEncoder(t *testing.T) {
}

func cycloneDxRedactor(s []byte) []byte {
serialPattern := regexp.MustCompile(`urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
rfc3339Pattern := regexp.MustCompile(`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`)
sha256Pattern := regexp.MustCompile(`sha256:[A-Fa-f0-9]{64}`)
for _, pattern := range []*regexp.Regexp{serialPattern, rfc3339Pattern, sha256Pattern} {
s = pattern.ReplaceAll(s, []byte(""))
replacements := map[string]string{
// UUIDs
`urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`: `urn:uuid:redacted`,
// timestamps
`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `timestamp:redacted`,
// image hashes
`sha256:[A-Fa-f0-9]{64}`: `sha256:redacted`,
// bom-refs
`"bom-ref":\s*"[^"]+"`: `"bom-ref": "redacted"`,
}
for pattern, replacement := range replacements {
s = regexp.MustCompile(pattern).ReplaceAll(s, []byte(replacement))
}
// the bom-ref will be autogenerated every time, the value here should not be directly tested in snapshot tests
s = regexp.MustCompile(`\s+"bom-ref":\s*"[^"]+",?\n`).ReplaceAll(s, []byte(""))

return s
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
"$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json",
"bomFormat": "CycloneDX",
"specVersion": "1.4",
"serialNumber": "urn:uuid:28fd9334-bfc4-403f-9335-9bcaa4b34e8b",
"serialNumber": "urn:uuid:redacted",
"version": 1,
"metadata": {
"timestamp": "2023-04-26T10:30:38-04:00",
"timestamp": "timestamp:redacted",
"tools": [
{
"vendor": "anchore",
Expand All @@ -14,14 +14,14 @@
}
],
"component": {
"bom-ref": "163686ac6e30c752",
"bom-ref": "redacted",
"type": "file",
"name": "/some/path"
}
},
"components": [
{
"bom-ref": "ed5945af814b9a95",
"bom-ref": "redacted",
"type": "library",
"name": "package-1",
"version": "1.0.1",
Expand Down Expand Up @@ -56,7 +56,7 @@
]
},
{
"bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=db4abfe497c180d3",
"bom-ref": "redacted",
"type": "library",
"name": "package-2",
"version": "2.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
"$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json",
"bomFormat": "CycloneDX",
"specVersion": "1.4",
"serialNumber": "urn:uuid:757286ea-45e3-4134-9ab5-8ac54f90688c",
"serialNumber": "urn:uuid:redacted",
"version": 1,
"metadata": {
"timestamp": "2023-04-26T10:30:29-04:00",
"timestamp": "timestamp:redacted",
"tools": [
{
"vendor": "anchore",
Expand All @@ -14,15 +14,15 @@
}
],
"component": {
"bom-ref": "38160ebc2a6876e8",
"bom-ref": "redacted",
"type": "container",
"name": "user-image-input",
"version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
"version": "sha256:redacted"
}
},
"components": [
{
"bom-ref": "ed77c5cb0567a1a6",
"bom-ref": "redacted",
"type": "library",
"name": "package-1",
"version": "1.0.1",
Expand Down Expand Up @@ -52,7 +52,7 @@
},
{
"name": "syft:location:0:layerID",
"value": "sha256:ab62016f9bec7286af65604081564cadeeb364a48faca2346c3f5a5a1f5ef777"
"value": "sha256:redacted"
},
{
"name": "syft:location:0:path",
Expand All @@ -61,7 +61,7 @@
]
},
{
"bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=958443e2d9304af4",
"bom-ref": "redacted",
"type": "library",
"name": "package-2",
"version": "2.0.1",
Expand All @@ -82,7 +82,7 @@
},
{
"name": "syft:location:0:layerID",
"value": "sha256:f1803845b6747d94d6e4ecce2331457e5f1c4fb97de5216f392a76f4582f63b2"
"value": "sha256:redacted"
},
{
"name": "syft:location:0:path",
Expand Down
Loading

0 comments on commit 67def34

Please sign in to comment.