Skip to content

Commit

Permalink
Merge pull request #224 from anchore/add_package_json
Browse files Browse the repository at this point in the history
Add support for package.json #200
  • Loading branch information
Toure Dunnon authored Oct 16, 2020
2 parents 4c751cb + 27c62e3 commit c0b9d78
Show file tree
Hide file tree
Showing 18 changed files with 857 additions and 109 deletions.
4 changes: 3 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ unit: fixtures ## Run unit tests (with coverage)
.PHONY: integration
integration: ## Run integration tests
$(call title,Running integration tests)
go test -tags=integration -v ./test/integration

go test -v ./test/integration


# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
integration-fingerprint:
Expand Down
9 changes: 9 additions & 0 deletions schema/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
## Updating the JSON schema
Today the JSON schema is generated from integration test data. Specifically, when integration tests are run, the `/schema/json/examples` directory is populated with syft JSON output data. This examples directory is used to drive automatically generating the JSON schema.
The caveats with this approach is:
1) the JSON schema is only as good as the examples provided
2) there is an integration test that ensures that the JSON schema is valid relative to what the code currently generates.
This means to update the JSON schema you need to
1) Open up `test/integration/json_schema_test.go` and comment out invocations of the `validateAgainstV1Schema` function.
2) From the root of the repo run `generate-json-schema`. Now there should be a new schema generated at `/schema/json/schema.json`
3) Uncomment the `validateAgainstV1Schema` function.
74 changes: 42 additions & 32 deletions schema/json/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,48 +40,58 @@
"architecture": {
"type": "string"
},
"author": {
"type": "string"
},
"description": {
"type": "string"
},
"epoch": {
"type": "integer"
},
"files": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"properties": {
"checksum": {
"type": "string"
},
"ownerGid": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"path": {
"anyOf": [
{
"type": "null"
},
{
"items": {
"anyOf": [
{
"type": "string"
},
"permissions": {
"type": "string"
{
"properties": {
"checksum": {
"type": "string"
},
"ownerGid": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"path": {
"type": "string"
},
"permissions": {
"type": "string"
}
},
"required": [
"checksum",
"ownerGid",
"ownerUid",
"path",
"permissions"
],
"type": "object"
}
},
"required": [
"checksum",
"ownerGid",
"ownerUid",
"path",
"permissions"
],
"type": "object"
}
]
},
"type": "array"
]
},
"type": "array"
}
]
},
"gitCommitOfApkPort": {
"type": "string"
Expand Down
8 changes: 4 additions & 4 deletions syft/cataloger/cataloger.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ type Cataloger interface {
func ImageCatalogers() []Cataloger {
return []Cataloger{
ruby.NewGemSpecCataloger(),
python.NewPythonCataloger(), // TODO: split and replace me
javascript.NewJavascriptCataloger(), // TODO: split and replace me
python.NewPythonCataloger(), // TODO: split and replace me
javascript.NewJavascriptPackageCataloger(),
deb.NewDpkgdbCataloger(),
rpmdb.NewRpmdbCataloger(),
java.NewJavaCataloger(),
Expand All @@ -51,8 +51,8 @@ func ImageCatalogers() []Cataloger {
func DirectoryCatalogers() []Cataloger {
return []Cataloger{
ruby.NewGemFileLockCataloger(),
python.NewPythonCataloger(), // TODO: split and replace me
javascript.NewJavascriptCataloger(), // TODO: split and replace me
python.NewPythonCataloger(), // TODO: split and replace me
javascript.NewJavascriptLockCataloger(),
deb.NewDpkgdbCataloger(),
rpmdb.NewRpmdbCataloger(),
java.NewJavaCataloger(),
Expand Down
15 changes: 12 additions & 3 deletions syft/cataloger/javascript/cataloger.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,21 @@ import (
"github.com/anchore/syft/syft/cataloger/common"
)

// NewJavascriptCataloger returns a new JavaScript cataloger object.
func NewJavascriptCataloger() *common.GenericCataloger {
// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
func NewJavascriptPackageCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package.json": parsePackageJSON,
}

return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
}

// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
func NewJavascriptLockCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package-lock.json": parsePackageLock,
"**/yarn.lock": parseYarnLock,
}

return common.NewGenericCataloger(nil, globParsers, "javascript-cataloger")
return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger")
}
54 changes: 54 additions & 0 deletions syft/cataloger/javascript/parse_package_json.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package javascript

import (
"encoding/json"
"fmt"
"io"

"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)

// integrity check
var _ common.ParserFn = parsePackageLock

// PackageJSON represents a JavaScript package.json file
type PackageJSON struct {
Version string `json:"version"`
Latest []string `json:"latest"`
Author string `json:"author"`
License string `json:"license"`
Name string `json:"name"`
Homepage string `json:"homepage"`
Description string `json:"description"`
Dependencies map[string]string `json:"dependencies"`
}

// parsePackageJson parses a package.json and returns the discovered JavaScript packages.
func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
dec := json.NewDecoder(reader)

for {
var p PackageJSON
if err := dec.Decode(&p); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
}

packages = append(packages, pkg.Package{
Name: p.Name,
Version: p.Version,
Licenses: []string{p.License},
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Metadata: pkg.NpmMetadata{
Author: p.Author,
Homepage: p.Homepage,
},
})
}

return packages, nil
}
43 changes: 43 additions & 0 deletions syft/cataloger/javascript/parse_package_json_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package javascript

import (
"os"
"testing"

"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)

func TestParsePackageJSON(t *testing.T) {
expected := pkg.Package{
Name: "npm",
Version: "6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
Metadata: pkg.NpmMetadata{
Author: "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
Homepage: "https://docs.npmjs.com/",
},
}
fixture, err := os.Open("test-fixtures/pkg-json/package.json")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}

actual, err := parsePackageJSON(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
if len(actual) != 1 {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=1", len(actual))
}

for _, d := range deep.Equal(actual[0], expected) {
t.Errorf("diff: %+v", d)
}

}
2 changes: 1 addition & 1 deletion syft/cataloger/javascript/parse_yarn_lock.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func parseYarnLock(_ string, reader io.Reader) ([]pkg.Package, error) {
Name: currentName,
Version: strings.Trim(version, "\""),
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
})
}
}
Expand Down
14 changes: 7 additions & 7 deletions syft/cataloger/javascript/parse_yarn_lock_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,43 @@ func TestParseYarnLock(t *testing.T) {
Name: "@babel/code-frame",
Version: "7.10.4",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
"@types/minimatch": {
Name: "@types/minimatch",
Version: "3.0.3",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
"@types/qs": {
Name: "@types/qs",
Version: "6.9.4",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
"ajv": {
Name: "ajv",
Version: "6.12.3",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
"atob": {
Name: "atob",
Version: "2.1.2",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
"aws-sdk": {
Name: "aws-sdk",
Version: "2.706.0",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
"jhipster-core": {
Name: "jhipster-core",
Version: "7.3.4",
Language: pkg.JavaScript,
Type: pkg.YarnPkg,
Type: pkg.NpmPkg,
},
}
fixture, err := os.Open("test-fixtures/yarn/yarn.lock")
Expand Down
Loading

0 comments on commit c0b9d78

Please sign in to comment.