Skip to content

Commit

Permalink
Add vulnerability and License batch querying (#2218)
Browse files Browse the repository at this point in the history 
* add new search query for vulns and licenses

Signed-off-by: pxp928 <parth.psu@gmail.com>

* update query and move to certvuln and certlegal, remove pagination

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add missing dec/dis license in bulk certifyvuln query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* update backend unit tests

Signed-off-by: pxp928 <parth.psu@gmail.com>

* remove static no vuln id and generate it during runtime

Signed-off-by: pxp928 <parth.psu@gmail.com>

* remove old comment

Signed-off-by: pxp928 <parth.psu@gmail.com>

---------

Signed-off-by: pxp928 <parth.psu@gmail.com>
  • Loading branch information
@pxp928
pxp928 authored Oct 28, 2024
1 parent 706f6d7 commit 6642687
Show file tree
Hide file tree
Showing 17 changed files with 692 additions and 7 deletions.
99 changes: 99 additions & 0 deletions internal/testing/backend/certifyLegal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,105 @@ func TestLegal(t *testing.T) {
}
}

func TestBatchQueryPkgIDCertifyLegal(t *testing.T) {
ctx := context.Background()
b := setupTest(t)
type call struct {
PkgSrc model.PackageOrSourceInputs
Dec [][]*model.IDorLicenseInput
Dis [][]*model.IDorLicenseInput
Legal []*model.CertifyLegalInputSpec
}
tests := []struct {
Name string
InPkg []*model.PkgInputSpec
InSrc []*model.SourceInputSpec
InLic []*model.LicenseInputSpec
Calls []call
ExpLegal []*model.CertifyLegal
}{
{
Name: "HappyPath",
InPkg: []*model.PkgInputSpec{testdata.P1, testdata.P2, testdata.P3, testdata.P4},
InLic: []*model.LicenseInputSpec{testdata.L1, testdata.L2, testdata.L3, testdata.L4},
Calls: []call{
{
PkgSrc: model.PackageOrSourceInputs{
Packages: []*model.IDorPkgInput{{PackageInput: testdata.P1}, {PackageInput: testdata.P2}, {PackageInput: testdata.P3}, {PackageInput: testdata.P4}},
},
Dec: [][]*model.IDorLicenseInput{{{LicenseInput: testdata.L1}}, {{LicenseInput: testdata.L2}}, {{LicenseInput: testdata.L3}}, {{LicenseInput: testdata.L4}}},
Dis: [][]*model.IDorLicenseInput{{{LicenseInput: testdata.L1}}, {{LicenseInput: testdata.L2}}, {}, {}},
Legal: []*model.CertifyLegalInputSpec{
{Justification: "test justification"},
{Justification: "test justification"},
{Justification: "test justification"},
{Justification: "test justification"},
},
},
},
ExpLegal: []*model.CertifyLegal{
{
Subject: testdata.P1out,
DeclaredLicenses: []*model.License{testdata.L1out},
DiscoveredLicenses: []*model.License{testdata.L1out},
Justification: "test justification",
},
{
Subject: testdata.P2out,
DeclaredLicenses: []*model.License{testdata.L2out},
DiscoveredLicenses: []*model.License{testdata.L2out},
Justification: "test justification",
},
{
Subject: testdata.P3out,
DeclaredLicenses: []*model.License{testdata.L3out},
Justification: "test justification",
},
{
Subject: testdata.P4out,
DeclaredLicenses: []*model.License{testdata.L4out},
Justification: "test justification",
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
var pkgIDs []string
for _, p := range test.InPkg {
if pkgID, err := b.IngestPackage(ctx, model.IDorPkgInput{PackageInput: p}); err != nil {
t.Fatalf("Could not ingest package: %v", err)
} else {
pkgIDs = append(pkgIDs, pkgID.PackageVersionID)
}
}
for _, s := range test.InSrc {
if _, err := b.IngestSource(ctx, model.IDorSourceInput{SourceInput: s}); err != nil {
t.Fatalf("Could not ingest source: %v", err)
}
}
for _, a := range test.InLic {
if _, err := b.IngestLicense(ctx, &model.IDorLicenseInput{LicenseInput: a}); err != nil {
t.Fatalf("Could not ingest license: %v", err)
}
}
for _, o := range test.Calls {
_, err := b.IngestCertifyLegals(ctx, o.PkgSrc, o.Dec, o.Dis, o.Legal)
if err != nil {
t.Fatalf("did not get expected ingest error: %v", err)
}
}
got, err := b.BatchQueryPkgIDCertifyLegal(ctx, pkgIDs)
if err != nil {
t.Fatalf("did not get expected query error: %v", err)
}
if diff := cmp.Diff(test.ExpLegal, got, commonOpts); diff != "" {
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
}
})
}
}

func TestLegals(t *testing.T) {
ctx := context.Background()
b := setupTest(t)
Expand Down
138 changes: 138 additions & 0 deletions internal/testing/backend/certifyVuln_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1516,6 +1516,144 @@ func TestIngestCertifyVulns(t *testing.T) {
}
}

func TestBatchQueryPkgIDCertifyVuln(t *testing.T) {
ctx := context.Background()
b := setupTest(t)
type call struct {
Pkgs []*model.IDorPkgInput
Vulns []*model.IDorVulnerabilityInput
CertifyVulns []*model.ScanMetadataInput
}
tests := []struct {
InPkg []*model.PkgInputSpec
Name string
InVuln []*model.VulnerabilityInputSpec
Calls []call
ExpVuln []*model.CertifyVuln
}{
{
Name: "HappyPath",
InVuln: []*model.VulnerabilityInputSpec{testdata.C1, testdata.C2, testdata.G1, testdata.O1},
InPkg: []*model.PkgInputSpec{testdata.P1, testdata.P2, testdata.P3, testdata.P4},
Calls: []call{
{
Pkgs: []*model.IDorPkgInput{{PackageInput: testdata.P2}, {PackageInput: testdata.P1}, {PackageInput: testdata.P3}, {PackageInput: testdata.P4}},
Vulns: []*model.IDorVulnerabilityInput{{VulnerabilityInput: testdata.C1}, {VulnerabilityInput: testdata.C2}, {VulnerabilityInput: testdata.G1}, {VulnerabilityInput: testdata.O1}},
CertifyVulns: []*model.ScanMetadataInput{
{
Collector: "test collector",
Origin: "test origin",
ScannerVersion: "v1.0.0",
ScannerURI: "test scanner uri",
DbVersion: "2023.01.01",
DbURI: "test db uri",
TimeScanned: testdata.T1,
},
{
Collector: "test collector",
Origin: "test origin",
ScannerVersion: "v1.0.0",
ScannerURI: "test scanner uri",
DbVersion: "2023.01.01",
DbURI: "test db uri",
TimeScanned: testdata.T1,
},
{
Collector: "test collector",
Origin: "test origin",
ScannerVersion: "v1.0.0",
ScannerURI: "test scanner uri",
DbVersion: "2023.01.01",
DbURI: "test db uri",
TimeScanned: testdata.T1,
},
{
Collector: "test collector",
Origin: "test origin",
ScannerVersion: "v1.0.0",
ScannerURI: "test scanner uri",
DbVersion: "2023.01.01",
DbURI: "test db uri",
TimeScanned: testdata.T1,
},
},
},
},
ExpVuln: []*model.CertifyVuln{
{
ID: "1",
Package: testdata.P2out,
Vulnerability: &model.Vulnerability{
Type: "cve",
VulnerabilityIDs: []*model.VulnerabilityID{testdata.C1out},
},
Metadata: vmd1,
},
{
ID: "10",
Package: testdata.P1out,
Vulnerability: &model.Vulnerability{
Type: "cve",
VulnerabilityIDs: []*model.VulnerabilityID{testdata.C2out},
},
Metadata: vmd1,
},
{
ID: "1",
Package: testdata.P3out,
Vulnerability: &model.Vulnerability{
Type: "ghsa",
VulnerabilityIDs: []*model.VulnerabilityID{testdata.G1out},
},
Metadata: vmd1,
},
{
ID: "1",
Package: testdata.P4out,
Vulnerability: &model.Vulnerability{
Type: "osv",
VulnerabilityIDs: []*model.VulnerabilityID{testdata.O1out},
},
Metadata: vmd1,
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
var pkgIDs []string
for _, v := range test.InVuln {
if _, err := b.IngestVulnerability(ctx, model.IDorVulnerabilityInput{VulnerabilityInput: v}); err != nil {
t.Fatalf("Could not ingest vulnerabilities: %a", err)
}
}
for _, p := range test.InPkg {
if pkgID, err := b.IngestPackage(ctx, model.IDorPkgInput{PackageInput: p}); err != nil {
t.Fatalf("Could not ingest packages: %v", err)
} else {
pkgIDs = append(pkgIDs, pkgID.PackageVersionID)
}
}
for _, o := range test.Calls {
_, err := b.IngestCertifyVulns(ctx, o.Pkgs, o.Vulns, o.CertifyVulns)
if err != nil {
t.Fatalf("did not get expected ingest error: %v", err)
}

}
got, err := b.BatchQueryPkgIDCertifyVuln(ctx, pkgIDs)
if err != nil {
t.Fatalf("did not get expected query error: %v", err)

}

if diff := cmp.Diff(test.ExpVuln, got, commonOpts); diff != "" {
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
}
})
}
}

func TestDeleteCertifyVuln(t *testing.T) {
ctx := context.Background()
b := setupTest(t)
Expand Down
10 changes: 6 additions & 4 deletions internal/testing/backend/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,12 @@ var skipMatrix = map[string]map[string]bool{
"TestVEXBulkIngest": {arango: true, redis: true},
"TestFindSoftware": {redis: true, arango: true},
// remove these once its implemented for the other backends
"TestDeleteCertifyVuln": {arango: true, memmap: true, redis: true, tikv: true},
"TestDeleteHasSBOM": {arango: true, memmap: true, redis: true, tikv: true},
"TestDeleteHasSLSAs": {arango: true, memmap: true, redis: true, tikv: true},
"TestQueryPackagesListForScan": {arango: true, redis: true, tikv: true},
"TestDeleteCertifyVuln": {arango: true, memmap: true, redis: true, tikv: true},
"TestDeleteHasSBOM": {arango: true, memmap: true, redis: true, tikv: true},
"TestDeleteHasSLSAs": {arango: true, memmap: true, redis: true, tikv: true},
"TestQueryPackagesListForScan": {arango: true, redis: true, tikv: true},
"TestBatchQueryPkgIDCertifyVuln": {arango: true, redis: true, tikv: true},
"TestBatchQueryPkgIDCertifyLegal": {arango: true, redis: true, tikv: true},
}

type backend interface {
Expand Down
30 changes: 30 additions & 0 deletions internal/testing/mocks/backend.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions pkg/assembler/backends/arangodb/search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@ func (c *arangoClient) FindPackagesThatNeedScanning(ctx context.Context, queryTy
return nil, fmt.Errorf("not implemented: FindPackagesThatNeedScanning")
}

func (c *arangoClient) BatchQueryPkgIDCertifyVuln(ctx context.Context, pkgIDs []string) ([]*model.CertifyVuln, error) {
return nil, fmt.Errorf("not implemented: BatchQueryPkgIDCertifyVuln")
}

func (c *arangoClient) BatchQueryPkgIDCertifyLegal(ctx context.Context, pkgIDs []string) ([]*model.CertifyLegal, error) {
return nil, fmt.Errorf("not implemented: BatchQueryPkgIDCertifyLegal")
}

// TODO(lumjjb): add source when it is implemented in arango backend
func (c *arangoClient) FindSoftware(ctx context.Context, searchText string) ([]model.PackageSourceOrArtifact, error) {

Expand Down
4 changes: 4 additions & 0 deletions pkg/assembler/backends/backends.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ type Backend interface {
Nodes(ctx context.Context, nodes []string) ([]model.Node, error)
Path(ctx context.Context, subject string, target string, maxPathLength int, usingOnly []model.Edge) ([]model.Node, error)

// Batch Query
BatchQueryPkgIDCertifyLegal(ctx context.Context, pkgIDs []string) ([]*model.CertifyLegal, error)
BatchQueryPkgIDCertifyVuln(ctx context.Context, pkgIDs []string) ([]*model.CertifyVuln, error)

// Search queries: queries to help find data in GUAC based on text search
FindSoftware(ctx context.Context, searchText string) ([]model.PackageSourceOrArtifact, error)
FindSoftwareList(ctx context.Context, searchText string, after *string, first *int) (*model.FindSoftwareConnection, error)
Expand Down
Loading

0 comments on commit 6642687

Please sign in to comment.