From f26d089b6d037fb39e88067d2b43a7cb3c352c32 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Mon, 26 Feb 2024 08:49:56 +0100 Subject: [PATCH 01/42] WIP: gather all properties in hierarchy Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 45 ++++++++++++++++++--- syft/pkg/cataloger/java/parse_pom_xml.go | 10 ++++- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index e84db154a4f..777380c0d46 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -28,8 +28,24 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque return requestURL, err } +// Add all properties that are not already in the map. +func addMissingProperties(properties map[string]string, pom gopom.Project) { + if pom.Properties != nil && pom.Properties.Entries != nil { + for name, value := range pom.Properties.Entries { + _, exists := properties[name] + if !exists { + properties[name] = value + log.Debugf(" Added property %s=%s", name, value) + } + } + } +} + // An artifact can have its version defined in a parent's DependencyManagement section -func recursivelyFindVersionFromParentPom(ctx context.Context, groupID, artifactID, parentGroupID, parentArtifactID, parentVersion string, cfg ArchiveCatalogerConfig) string { +func recursivelyFindVersionFromParentPom(ctx context.Context, groupID, artifactID, parentGroupID, parentArtifactID, parentVersion string, cfg ArchiveCatalogerConfig) (string, map[string]string) { + log.Debugf("recursively finding version from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", + groupID, artifactID, parentGroupID, parentArtifactID, parentVersion) + var projectProperties map[string]string = make(map[string]string) // As there can be nested parent poms, we'll recursively check for the version until we reach the max depth for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { parentPom, err := getPomFromMavenRepo(ctx, parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) @@ -38,10 +54,22 @@ func recursivelyFindVersionFromParentPom(ctx context.Context, groupID, artifactI log.Tracef("unable to get parent pom from Maven central: %v", err) break } - if parentPom != nil && parentPom.DependencyManagement != nil { - for _, dependency := range *parentPom.DependencyManagement.Dependencies { - if groupID == *dependency.GroupID && artifactID == *dependency.ArtifactID && dependency.Version != nil { - return *dependency.Version + if parentPom != nil { + addMissingProperties(projectProperties, *parentPom) + if parentPom.DependencyManagement != nil { + log.Debug("Here") + for _, dependency := range *parentPom.DependencyManagement.Dependencies { + // imported pom files should be treated just like parent poms, they are use to define versions of dependencies + if dependency.Type != nil && dependency.Scope != nil && + *dependency.Type == "pom" && *dependency.Scope == "import" { + depVersion := resolveProperty(*parentPom, dependency.Version, "version") + foundVersion, properties := recursivelyFindVersionFromParentPom(ctx, groupID, artifactID, *dependency.GroupID, *dependency.ArtifactID, depVersion, cfg) + return foundVersion, properties + } + if groupID == *dependency.GroupID && artifactID == *dependency.ArtifactID && dependency.Version != nil { + version := resolveProperty(*parentPom, dependency.Version, "version") + return version, projectProperties + } } } } @@ -52,10 +80,12 @@ func recursivelyFindVersionFromParentPom(ctx context.Context, groupID, artifactI parentArtifactID = *parentPom.Parent.ArtifactID parentVersion = *parentPom.Parent.Version } - return "" + return "", projectProperties } func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { + log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", + groupID, artifactID, groupID, artifactID, version) var licenses []string // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { @@ -80,6 +110,9 @@ func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifact } func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { + if len(groupID) == 0 || len(artifactID) == 0 || len(version) == 0 || strings.HasPrefix(version, "${") { + return nil, fmt.Errorf("missing/incomplete maven artifact coordinates: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) + } requestURL, err := formatMavenPomURL(groupID, artifactID, version, mavenBaseURL) if err != nil { return nil, err diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 64b8d614a5d..8615856bf4d 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -111,14 +111,18 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende name := safeString(dep.ArtifactID) version := resolveProperty(pom, dep.Version, "version") + var properties map[string]string licenses := make([]pkg.License, 0) if cfg.UseNetwork { if version == "" { // If we have no version then let's try to get it from a parent pom DependencyManagement section - version = recursivelyFindVersionFromParentPom(ctx, *dep.GroupID, *dep.ArtifactID, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg) + version, properties = recursivelyFindVersionFromParentPom(ctx, *dep.GroupID, *dep.ArtifactID, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg) + pom.Properties.Entries = properties + version = resolveProperty(pom, &version, "version") } if version != "" { + parentLicenses := recursivelyFindLicensesFromParentPom( ctx, m.PomProperties.GroupID, @@ -134,6 +138,10 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende } } + if strings.HasPrefix(version, "${") { + log.Warnf("Got version '%s' for artifact: %s", version, name) + } + p := pkg.Package{ Name: name, Version: version, From e96dbb1ca34befeed5cf0639c7d3230d0acaf4bf Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Wed, 28 Feb 2024 22:18:15 +0100 Subject: [PATCH 02/42] WIP: recurse into parents and boms Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 223 ++++++++++++++++---- syft/pkg/cataloger/java/parse_pom_xml.go | 71 ++++++- 2 files changed, 251 insertions(+), 43 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 777380c0d46..5b72cca5050 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -28,59 +28,208 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque return requestURL, err } -// Add all properties that are not already in the map. -func addMissingProperties(properties map[string]string, pom gopom.Project) { - if pom.Properties != nil && pom.Properties.Entries != nil { +// Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. +func addMissingPropertiesToProject(allProperties map[string]string, pom *gopom.Project) { + if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { for name, value := range pom.Properties.Entries { - _, exists := properties[name] + _, exists := allProperties[name] if !exists { - properties[name] = value - log.Debugf(" Added property %s=%s", name, value) + allProperties[name] = value + // log.Tracef(" Added property %s=%s to pom", name, value) } } + } else { + log.Tracef("addMissingPropertiesToProject: nothing to do for project: %s", *pom.ArtifactID) } } -// An artifact can have its version defined in a parent's DependencyManagement section -func recursivelyFindVersionFromParentPom(ctx context.Context, groupID, artifactID, parentGroupID, parentArtifactID, parentVersion string, cfg ArchiveCatalogerConfig) (string, map[string]string) { - log.Debugf("recursively finding version from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", - groupID, artifactID, parentGroupID, parentArtifactID, parentVersion) - var projectProperties map[string]string = make(map[string]string) - // As there can be nested parent poms, we'll recursively check for the version until we reach the max depth - for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { - parentPom, err := getPomFromMavenRepo(ctx, parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) - if err != nil { - // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - log.Tracef("unable to get parent pom from Maven central: %v", err) - break +// Add all properties from the map 'additionalProperties' to the map 'allProperties' that are not already in the map. +func addMissingPropertiesToMap(allProperties, additionalProperties map[string]string) { + if len(additionalProperties) > 0 { + for name, value := range additionalProperties { + _, exists := additionalProperties[name] + if !exists { + allProperties[name] = value + // log.Tracef(" Added property to allProperties %s=%s", name, value) + } + } + } else { + log.Tracef("addMissingPropertiesToMap: Supplied map was empty.") + } +} + +// Add all properties from map 'properties' to the project 'pom' that are not already defined in the pom. +// This increases the chance of the 'resolveProperty' function succeeding. +func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) { + + if len(allProperties) > 0 { + if pom.Properties == nil { + var props gopom.Properties + props.Entries = make(map[string]string) + pom.Properties = &props + } + for name, value := range allProperties { + _, exists := pom.Properties.Entries[name] + if !exists { + allProperties[name] = value + log.Tracef(" Added property %s=%s to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) + } } + } +} + +// Try to find the version of a dependency (groupID, artifactID) by parsing all parent poms and imported managed dependencies (maven BOMs). +// Properties are gathered in the order that they are encountered: in Maven the latest definition of a property (highest in hierarchy) is used. +// parsedPomFiles contains all previously parsed pom files encountered by earlier invocations of this function on the stack. So for the first +// call parsedPomFiles should be nill. It is used to prevent cycles (endless loops). +func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, + pom *gopom.Project, cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) (string, map[string]string) { + + log.Debugf("recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]", + findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version) + + // Create map to keep track of parsed pom files and to prevent cycles. + if parsedPomFiles == nil { + parsedPomFiles = make(map[MavenCoordinate]bool) + log.Tracef("Created parsedPomFiles") + } + log.Tracef("parsedPomFiles: %+v", parsedPomFiles) + + pomCoordinates := MavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} + _, alreadyParsed := parsedPomFiles[pomCoordinates] + if alreadyParsed { + // Nothing new here, already parsed + log.Info("Nothing new here, already parsed.") + return "", nil + } else { + parsedPomFiles[pomCoordinates] = true + } + + // Map with all properties defined in all parsed pom files + var allProperties map[string]string = make(map[string]string) + + addMissingPropertiesToProject(allProperties, pom) + + // If a parent exists, first parse the parent POM. It may contain required properties and/or + // managed dependencies. + if pom.Parent != nil { + parentGroupID := *pom.Parent.GroupID + parentArtifactID := *pom.Parent.ArtifactID + parentVersion := *pom.Parent.Version + + parentPom, err := getPomFromMavenOrCache(ctx, parentGroupID, parentArtifactID, parentVersion, cfg) + if parentPom != nil { - addMissingProperties(projectProperties, *parentPom) + log.Infof("Found a parent pom: [%s, %s, %s]", parentGroupID, parentArtifactID, parentVersion) + // Mark this parent pom as parsed to prevent re-parsing/cycles later on + parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] = parentPom + + addMissingPropertiesToProject(allProperties, parentPom) + addPropertiesToProject(parentPom, allProperties) + + // TODO: Recurse into parent to gather properties/dep management? + + foundVersion := "" if parentPom.DependencyManagement != nil { - log.Debug("Here") - for _, dependency := range *parentPom.DependencyManagement.Dependencies { - // imported pom files should be treated just like parent poms, they are use to define versions of dependencies - if dependency.Type != nil && dependency.Scope != nil && - *dependency.Type == "pom" && *dependency.Scope == "import" { - depVersion := resolveProperty(*parentPom, dependency.Version, "version") - foundVersion, properties := recursivelyFindVersionFromParentPom(ctx, groupID, artifactID, *dependency.GroupID, *dependency.ArtifactID, depVersion, cfg) - return foundVersion, properties - } - if groupID == *dependency.GroupID && artifactID == *dependency.ArtifactID && dependency.Version != nil { - version := resolveProperty(*parentPom, dependency.Version, "version") - return version, projectProperties + foundVersion = findVersionInDependencyManagement( + ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, parsedPomFiles) + } + if parentPom.Dependencies != nil { + foundVersion = findVersionInDependencies(findGroupID, findArtifactID, parentPom) + } + + if foundVersion != "" && !strings.HasPrefix(foundVersion, "${}") { + foundVersion := resolveProperty(*parentPom, &foundVersion, "version") + log.Infof("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + return foundVersion, allProperties + } + } else { + log.Warnf("unable to get parent pom [%s, %s, %s]: %v", + parentGroupID, parentArtifactID, parentVersion, err) + } + } + + foundVersion := "" + if pom.DependencyManagement != nil { + foundVersion = findVersionInDependencyManagement( + ctx, findGroupID, findArtifactID, pom, cfg, allProperties, parsedPomFiles) + } + if pom.Dependencies != nil { + foundVersion = findVersionInDependencies(findGroupID, findArtifactID, pom) + } + + log.Infof("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + return foundVersion, allProperties +} + +// Get a parent pom from cache or download from a Maven repository +func getPomFromMavenOrCache(ctx context.Context, parentGroupID, parentArtifactID, parentVersion string, + cfg ArchiveCatalogerConfig) (*gopom.Project, error) { + var err error + parentPom, found := parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] + + if !found && cfg.UseNetwork { + parentPom, err = getPomFromMavenRepo(ctx, parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) + } + return parentPom, err +} + +// Find given dependency (groupID, artifactID) in the dependencyManagement section of project 'pom'. +// May recursively call recursivelyFindVersionFromManagedOrInherited when a Maven BOM is found. +func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, + pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { + + for _, dependency := range *pom.DependencyManagement.Dependencies { + log.Tracef(" Found managed dependency: [%s, %s, %s]", + safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) + + // imported pom files should be treated just like parent poms, they are use to define versions of dependencies + if dependency.Type != nil && dependency.Scope != nil && + *dependency.Type == "pom" && *dependency.Scope == "import" { + + bomVersion := resolveProperty(*pom, dependency.Version, "version") + log.Debugf("Found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) + // Recurse into BOM, which should be treated just like a parent pom + bomProject, err := getPomFromMavenOrCache(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, cfg) + if err == nil { + foundVersion, bomProperties := recursivelyFindVersionFromManagedOrInherited(ctx, findGroupID, findArtifactID, bomProject, cfg, parsedPomFiles) + log.Debugf("Finished processing BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) + addMissingPropertiesToMap(allProperties, bomProperties) + addMissingPropertiesToProject(allProperties, pom) + if foundVersion != "" { + foundVersion = resolveProperty(*pom, dependency.Version, "version") + if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { + log.Tracef("Found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) + return foundVersion } } } + + } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { + foundVersion := resolveProperty(*pom, dependency.Version, "version") + if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { + log.Tracef("Found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) + return foundVersion + } } - if parentPom == nil || parentPom.Parent == nil { - break + } + log.Tracef("Dependency not found in dependencyManagement") + return "" +} + +// Find given dependency (groupID, artifactID) in the dependencies section of project 'pom'. +func findVersionInDependencies(groupID, artifactID string, pom *gopom.Project) string { + + for _, dependency := range *pom.Dependencies { + if *dependency.GroupID == groupID && *dependency.ArtifactID == artifactID { + depVersion := resolveProperty(*pom, dependency.Version, "version") + // TODO: -> trace + log.Infof("Found dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, depVersion) + return depVersion } - parentGroupID = *parentPom.Parent.GroupID - parentArtifactID = *parentPom.Parent.ArtifactID - parentVersion = *parentPom.Parent.Version } - return "", projectProperties + log.Tracef("Dependency not found in dependencies") + return "" } func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 8615856bf4d..1e8b442c227 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -21,10 +21,26 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) +// MavenCoordinate is the unique identifier for a package in Maven. +type MavenCoordinate struct { + GroupID string + ArtifactID string + Version string +} + +// GroupId and ArtifactId of (managed) dependencies +type GroupArtifact struct { + GroupID string + ArtifactID string +} + const pomXMLGlob = "*pom.xml" var propertyMatcher = regexp.MustCompile("[$][{][^}]+[}]") +// Map containing all pom.xml files that have been parsed. Used for caching as pom files (might) have been downloaded from the internet +var parsedPomFilesCache map[MavenCoordinate]*gopom.Project = make(map[MavenCoordinate]*gopom.Project) + func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) if err != nil { @@ -46,6 +62,10 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. } pkgs = append(pkgs, p) + + if len(p.Version) == 0 || strings.HasPrefix(p.Version, "${") { + log.Infof("Found artifact without version: %s:%s, version: %q", *dep.GroupID, *dep.ArtifactID, p.Version) + } } } @@ -101,24 +121,33 @@ func newPomProject(path string, p gopom.Project, location file.Location) *parsed } func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Dependency, cfg ArchiveCatalogerConfig, locations ...file.Location) pkg.Package { + groupId := resolveProperty(pom, dep.GroupID, "groupId") + artifactId := resolveProperty(pom, dep.ArtifactID, "artifactId") + m := pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ - GroupID: resolveProperty(pom, dep.GroupID, "groupId"), - ArtifactID: resolveProperty(pom, dep.ArtifactID, "artifactId"), + GroupID: groupId, + ArtifactID: artifactId, Scope: resolveProperty(pom, dep.Scope, "scope"), }, } name := safeString(dep.ArtifactID) version := resolveProperty(pom, dep.Version, "version") - var properties map[string]string + log.Infof("New dependency: [%s, %s, %s].", groupId, artifactId, version) + var allProperties map[string]string licenses := make([]pkg.License, 0) if cfg.UseNetwork { if version == "" { // If we have no version then let's try to get it from a parent pom DependencyManagement section - version, properties = recursivelyFindVersionFromParentPom(ctx, *dep.GroupID, *dep.ArtifactID, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg) - pom.Properties.Entries = properties + version, allProperties = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, nil) + pom.Properties.Entries = allProperties + version = resolveProperty(pom, &version, "version") + } else if strings.HasPrefix(version, "${") { + // If we are missing the property for this version, search the pom hierarchy for it. + _, allProperties = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, nil) + pom.Properties.Entries = allProperties version = resolveProperty(pom, &version, "version") } if version != "" { @@ -135,6 +164,8 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } } + } else { + log.Warnf("Could not determine version for package: [%s, %s]", groupId, artifactId) } } @@ -172,6 +203,26 @@ func decodePomXML(content io.Reader) (project gopom.Project, err error) { return project, fmt.Errorf("unable to unmarshal pom.xml: %w", err) } + // For modules groupID and version are almost always inherited from parent pom + if project.GroupID == nil && project.Parent != nil { + project.GroupID = project.Parent.GroupID + } + if project.Version == nil && project.Parent != nil { + project.Version = project.Parent.Version + } + + // If missing, add maven built-in version property often used in multi-module projects + if project.Version != nil { + if project.Properties == nil { + var props gopom.Properties + props.Entries = make(map[string]string) + props.Entries["project.version"] = *project.Version + project.Properties = &props + } else { + project.Properties.Entries["project.version"] = *project.Version + } + } + return project, nil } @@ -242,11 +293,17 @@ func cleanDescription(original *string) (cleaned string) { //nolint:gocognit func resolveProperty(pom gopom.Project, property *string, propertyName string) string { propertyCase := safeString(property) + if !strings.Contains(propertyCase, "${") { + //nothing to resolve + // log.Tracef("resolving property: value [%s] contains no variable", propertyName) + return propertyCase + } log.WithFields("existingPropertyValue", propertyCase, "propertyName", propertyName).Trace("resolving property") return propertyMatcher.ReplaceAllStringFunc(propertyCase, func(match string) string { propertyName := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } entries := pomProperties(pom) if value, ok := entries[propertyName]; ok { + log.WithFields("existingPropertyValue", value, "propertyName", propertyName).Trace("resolved property") return value } @@ -287,7 +344,9 @@ func resolveProperty(pom gopom.Project, property *string, propertyName string) s } // If this was the last part of the property name, return the value if partNum == numParts-1 { - return fmt.Sprintf("%v", pomValue.Interface()) + value := fmt.Sprintf("%v", pomValue.Interface()) + log.WithFields("existingPropertyValue", value, "propertyName", propertyName).Trace("resolved property") + return value } break } From acdf111dd7316ace52fac0408d8528b08368d0bc Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:28:16 +0100 Subject: [PATCH 03/42] WIP: Getting there, rework of recursion needed Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 190 ++++++++++++++++---- syft/pkg/cataloger/java/parse_pom_xml.go | 47 +++-- 2 files changed, 184 insertions(+), 53 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 5b72cca5050..66fdece3394 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -29,7 +29,7 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque } // Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. -func addMissingPropertiesToProject(allProperties map[string]string, pom *gopom.Project) { +func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom.Project) { if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { for name, value := range pom.Properties.Entries { _, exists := allProperties[name] @@ -58,7 +58,7 @@ func addMissingPropertiesToMap(allProperties, additionalProperties map[string]st } } -// Add all properties from map 'properties' to the project 'pom' that are not already defined in the pom. +// Add all properties from map 'allProperties' to the project 'pom' that are not already defined in the pom. // This increases the chance of the 'resolveProperty' function succeeding. func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) { @@ -72,12 +72,52 @@ func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) _, exists := pom.Properties.Entries[name] if !exists { allProperties[name] = value - log.Tracef(" Added property %s=%s to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) + // log.Tracef(" Added property %s=%s to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) } } } } +// Traverse the parent pom hierarchy and return all found properties. +// To be used for resolving property variables. +func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]string, parentGroupID, parentArtifactID, parentVersion string, + cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) { + log.Debugf("Recursively gathering all properties from pom [%s, %s, %s]", parentGroupID, parentArtifactID, parentVersion) + + // Create map to keep track of parsed pom files and to prevent cycles. + if parsedPomFiles == nil { + parsedPomFiles = make(map[MavenCoordinate]bool) + } + log.Tracef("Recursion depth: %+v", len(parsedPomFiles)) + + pomCoordinates := MavenCoordinate{parentGroupID, parentArtifactID, parentVersion} + _, alreadyParsed := parsedPomFiles[pomCoordinates] + if alreadyParsed { + // Nothing new here, already parsed + log.Info("Nothing new here, already processed.") + return + } + + parentPom, err := getPomFromMavenOrCache(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) + + if err == nil { + if parentPom != nil { + parsedPomFiles[pomCoordinates] = true + addMissingPropertiesFromProject(allProperties, parentPom) + + // recurse into another parent pom + if parentPom.Parent != nil { + getPropertiesFromParentPoms(ctx, allProperties, *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version, + cfg, parsedPomFiles) + } + } else { + log.Error("Got empty parent pom, error: %w") + } + } else { + log.Errorf("Could not get parent pom: %w", err) + } +} + // Try to find the version of a dependency (groupID, artifactID) by parsing all parent poms and imported managed dependencies (maven BOMs). // Properties are gathered in the order that they are encountered: in Maven the latest definition of a property (highest in hierarchy) is used. // parsedPomFiles contains all previously parsed pom files encountered by earlier invocations of this function on the stack. So for the first @@ -85,21 +125,20 @@ func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, pom *gopom.Project, cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) (string, map[string]string) { - log.Debugf("recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]", + log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]", findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version) // Create map to keep track of parsed pom files and to prevent cycles. if parsedPomFiles == nil { parsedPomFiles = make(map[MavenCoordinate]bool) - log.Tracef("Created parsedPomFiles") } - log.Tracef("parsedPomFiles: %+v", parsedPomFiles) + log.Tracef("Recursion depth: %+v", len(parsedPomFiles)) pomCoordinates := MavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { // Nothing new here, already parsed - log.Info("Nothing new here, already parsed.") + log.Info("Nothing new here, already processed.") return "", nil } else { parsedPomFiles[pomCoordinates] = true @@ -108,7 +147,7 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup // Map with all properties defined in all parsed pom files var allProperties map[string]string = make(map[string]string) - addMissingPropertiesToProject(allProperties, pom) + addMissingPropertiesFromProject(allProperties, pom) // If a parent exists, first parse the parent POM. It may contain required properties and/or // managed dependencies. @@ -117,32 +156,46 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup parentArtifactID := *pom.Parent.ArtifactID parentVersion := *pom.Parent.Version - parentPom, err := getPomFromMavenOrCache(ctx, parentGroupID, parentArtifactID, parentVersion, cfg) + parentPom, err := getPomFromMavenOrCache(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) if parentPom != nil { - log.Infof("Found a parent pom: [%s, %s, %s]", parentGroupID, parentArtifactID, parentVersion) + log.Infof("Found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) // Mark this parent pom as parsed to prevent re-parsing/cycles later on - parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] = parentPom + parsedPomFilesCache[MavenCoordinate{*parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version}] = parentPom - addMissingPropertiesToProject(allProperties, parentPom) + addMissingPropertiesFromProject(allProperties, parentPom) addPropertiesToProject(parentPom, allProperties) - // TODO: Recurse into parent to gather properties/dep management? + // TODO: Recurse into parent to gather properties (searching dependencies of parent poms) + // TODO: create function to just gather properties from all parents + // _, parentProperties := recursivelyFindVersionFromManagedOrInherited( + // ctx, findGroupID, findArtifactID, parentPom, cfg, parsedPomFiles) + // log.Info("return 1") + // addMissingPropertiesToMap(allProperties, parentProperties) + // addPropertiesToProject(parentPom, allProperties) foundVersion := "" - if parentPom.DependencyManagement != nil { - foundVersion = findVersionInDependencyManagement( - ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, parsedPomFiles) - } + if parentPom.Dependencies != nil { foundVersion = findVersionInDependencies(findGroupID, findArtifactID, parentPom) } - - if foundVersion != "" && !strings.HasPrefix(foundVersion, "${}") { - foundVersion := resolveProperty(*parentPom, &foundVersion, "version") - log.Infof("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + if foundVersion == "" && parentPom.DependencyManagement != nil { + foundVersion = findVersionInDependencyManagement( + ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, parsedPomFiles) + log.Infof("====> 1 Here: %s", foundVersion) + } + log.Infof("====> 2 Here: %s", foundVersion) + foundVersion = resolveProperty(*parentPom, &foundVersion, "version") + log.Infof("====> 3 Here: %s", foundVersion) + if isPropertyResolved(foundVersion) { + log.Infof("1Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) return foundVersion, allProperties } + // if foundVersion != "" || strings.HasPrefix(foundVersion, "${}") { + // foundVersion := resolveProperty(*parentPom, &foundVersion, "version") + // log.Infof("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + // return foundVersion, allProperties + // } } else { log.Warnf("unable to get parent pom [%s, %s, %s]: %v", parentGroupID, parentArtifactID, parentVersion, err) @@ -158,18 +211,30 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup foundVersion = findVersionInDependencies(findGroupID, findArtifactID, pom) } - log.Infof("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + if foundVersion == "" { + log.Infof("No version found for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + } else { + log.Infof("2Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + } return foundVersion, allProperties } +// Returns true when value is not empty and does not start with "${" (contains an unresolved property). +func isPropertyResolved(value string) bool { + return value != "" && !strings.HasPrefix(value, "${}") +} + // Get a parent pom from cache or download from a Maven repository -func getPomFromMavenOrCache(ctx context.Context, parentGroupID, parentArtifactID, parentVersion string, +func getPomFromMavenOrCache(ctx context.Context, parentGroupID, parentArtifactID, parentVersion string, allProperties map[string]string, cfg ArchiveCatalogerConfig) (*gopom.Project, error) { - var err error + var err error = nil parentPom, found := parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] if !found && cfg.UseNetwork { parentPom, err = getPomFromMavenRepo(ctx, parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) + if err == nil { + addPropertiesToProject(parentPom, allProperties) + } } return parentPom, err } @@ -179,9 +244,9 @@ func getPomFromMavenOrCache(ctx context.Context, parentGroupID, parentArtifactID func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { - for _, dependency := range *pom.DependencyManagement.Dependencies { - log.Tracef(" Found managed dependency: [%s, %s, %s]", - safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) + for _, dependency := range *getPomManagedDependencies(pom) { + // log.Tracef(" Found managed dependency: [%s, %s, %s]", + // safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) // imported pom files should be treated just like parent poms, they are use to define versions of dependencies if dependency.Type != nil && dependency.Scope != nil && @@ -190,12 +255,17 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt bomVersion := resolveProperty(*pom, dependency.Version, "version") log.Debugf("Found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) // Recurse into BOM, which should be treated just like a parent pom - bomProject, err := getPomFromMavenOrCache(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, cfg) + bomProject, err := getPomFromMavenOrCache(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) if err == nil { foundVersion, bomProperties := recursivelyFindVersionFromManagedOrInherited(ctx, findGroupID, findArtifactID, bomProject, cfg, parsedPomFiles) - log.Debugf("Finished processing BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) + log.Info("return 2") + log.Debugf("Finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) addMissingPropertiesToMap(allProperties, bomProperties) - addMissingPropertiesToProject(allProperties, pom) + addMissingPropertiesFromProject(allProperties, pom) + if isPropertyResolved(foundVersion) { + log.Infof("---> HERE!: %s", foundVersion) + return foundVersion + } if foundVersion != "" { foundVersion = resolveProperty(*pom, dependency.Version, "version") if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { @@ -241,7 +311,7 @@ func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifact parentPom, err := getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) if err != nil { // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - log.Tracef("unable to get parent pom from Maven central: %v", err) + log.Tracef("unable to get parent pom from Maven repository: %v", err) return []string{} } parentLicenses := parseLicensesFromPom(parentPom) @@ -266,7 +336,7 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave if err != nil { return nil, err } - log.Tracef("trying to fetch parent pom from Maven central %s", requestURL) + log.Tracef("trying to fetch parent pom from Maven repository %s", requestURL) mavenRequest, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { @@ -281,7 +351,7 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave resp, err := httpClient.Do(mavenRequest) if err != nil { - return nil, fmt.Errorf("unable to get pom from Maven central: %w", err) + return nil, fmt.Errorf("unable to get pom from Maven repository: %w", err) } defer func() { if err := resp.Body.Close(); err != nil { @@ -291,12 +361,12 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave bytes, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven central: %w", err) + return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) } pom, err := decodePomXML(strings.NewReader(string(bytes))) if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven central: %w", err) + return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) } return &pom, nil @@ -316,3 +386,53 @@ func parseLicensesFromPom(pom *gopom.Project) []string { return licenses } + +// Returns all dependencies in a project, including all defined in profiles. +func getPomDependencies(pom *gopom.Project) *[]gopom.Dependency { + var dependencies []gopom.Dependency = make([]gopom.Dependency, 0) + if pom.Profiles != nil && len(*pom.Profiles) > 0 { + // Gather dependencies from profiles and main dependencies + if pom.Dependencies != nil { + dependencies = append(dependencies, *pom.Dependencies...) + } + + for _, profile := range *pom.Profiles { + if profile.Dependencies != nil { + dependencies = append(dependencies, *profile.Dependencies...) + } + } + return &dependencies + + } else { + if pom.Dependencies != nil { + return pom.Dependencies + } else { + return &dependencies + } + } +} + +// Returns all managed dependencies in a project, including all defined in profiles. +func getPomManagedDependencies(pom *gopom.Project) *[]gopom.Dependency { + if pom.Profiles != nil && len(*pom.Profiles) > 0 { + var mDependencies []gopom.Dependency = make([]gopom.Dependency, 0) + if pom.DependencyManagement != nil && pom.DependencyManagement.Dependencies != nil { + mDependencies = append(mDependencies, *pom.DependencyManagement.Dependencies...) + } + + for _, profile := range *pom.Profiles { + if profile.DependencyManagement != nil && profile.DependencyManagement.Dependencies != nil { + mDependencies = append(mDependencies, *profile.DependencyManagement.Dependencies...) + } + } + return &mDependencies + + } else { + if pom.DependencyManagement != nil && pom.DependencyManagement.Dependencies != nil { + return pom.DependencyManagement.Dependencies + } else { + var mDependencies []gopom.Dependency = make([]gopom.Dependency, 0) + return &mDependencies + } + } +} diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 1e8b442c227..aa2f7bf3f3e 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -48,24 +48,31 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. } var pkgs []pkg.Package - if pom.Dependencies != nil { - for _, dep := range *pom.Dependencies { - p := newPackageFromPom( - ctx, - pom, - dep, - gap.cfg, - reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), - ) - if p.Name == "" { - continue - } - pkgs = append(pkgs, p) + // Add all properties defined in parent poms to this project for resolving properties. + if pom.Parent != nil { + var allProperties map[string]string = make(map[string]string) + getPropertiesFromParentPoms( + ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, gap.cfg, nil) + addPropertiesToProject(&pom, allProperties) + } - if len(p.Version) == 0 || strings.HasPrefix(p.Version, "${") { - log.Infof("Found artifact without version: %s:%s, version: %q", *dep.GroupID, *dep.ArtifactID, p.Version) - } + for _, dep := range *getPomDependencies(&pom) { + p := newPackageFromPom( + ctx, + pom, + dep, + gap.cfg, + reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), + ) + if p.Name == "" { + continue + } + + pkgs = append(pkgs, p) + + if len(p.Version) == 0 || strings.HasPrefix(p.Version, "${") { + log.Infof("Found artifact without version: %s:%s, version: %q", *dep.GroupID, *dep.ArtifactID, p.Version) } } @@ -142,12 +149,16 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende if version == "" { // If we have no version then let's try to get it from a parent pom DependencyManagement section version, allProperties = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, nil) + log.Info("return 3") pom.Properties.Entries = allProperties version = resolveProperty(pom, &version, "version") } else if strings.HasPrefix(version, "${") { + log.Error("THIS SHOULD NOT HAPPEN!") // If we are missing the property for this version, search the pom hierarchy for it. - _, allProperties = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, nil) - pom.Properties.Entries = allProperties + if pom.Parent != nil { + getPropertiesFromParentPoms(ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, + cfg, nil) + } version = resolveProperty(pom, &version, "version") } if version != "" { From a0dd2ef9848bf6c1324084b68adbc2ba875f767c Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:11:29 +0100 Subject: [PATCH 04/42] Added resolving properties by name (reference value) Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 140 +++++++++++--------- syft/pkg/cataloger/java/parse_pom_xml.go | 37 ++++-- 2 files changed, 98 insertions(+), 79 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 66fdece3394..c3325b649aa 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -28,16 +28,54 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque return requestURL, err } +func resolveRecursiveByPropertyName(pomProperties map[string]string, propertyName string) string { + if strings.HasPrefix(propertyName, "${") { + name := getPropertyName(propertyName) + // log.Debugf("--Name: %s", name) + if value, ok := pomProperties[name]; ok { + // log.Debugf("--Value: %s", value) + if strings.HasPrefix(value, "${") { + log.Trace("recurse resolveRecursiveByPropertyName") + return resolveRecursiveByPropertyName(pomProperties, value) + } else { + // log.Debugf("++Value: %s", value) + return value + } + } + // log.Debugf("**Value: %s", propertyName) + return propertyName + } else { + // log.Debugf("&&Value: %s", propertyName) + return propertyName + } +} + +func getPropertyName(value string) string { + propertyName := value + if strings.HasPrefix(propertyName, "${") { + propertyName = strings.TrimSpace(propertyName[2 : len(propertyName)-1]) // remove leading ${ and trailing } + } + return propertyName +} + // Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom.Project) { if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { for name, value := range pom.Properties.Entries { _, exists := allProperties[name] if !exists { + value = resolveProperty(*pom, &value, getPropertyName(value)) allProperties[name] = value - // log.Tracef(" Added property %s=%s to pom", name, value) + log.Tracef(" Added from project, property %s=%s to allProperties", name, value) + } + } + // resolve + for name, value := range allProperties { + if strings.HasPrefix(value, "${") { + allProperties[name] = resolveRecursiveByPropertyName(allProperties, value) } } + } else { log.Tracef("addMissingPropertiesToProject: nothing to do for project: %s", *pom.ArtifactID) } @@ -94,7 +132,7 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { // Nothing new here, already parsed - log.Info("Nothing new here, already processed.") + log.Info("1 Nothing new here, already processed.") return } @@ -121,9 +159,9 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s // Try to find the version of a dependency (groupID, artifactID) by parsing all parent poms and imported managed dependencies (maven BOMs). // Properties are gathered in the order that they are encountered: in Maven the latest definition of a property (highest in hierarchy) is used. // parsedPomFiles contains all previously parsed pom files encountered by earlier invocations of this function on the stack. So for the first -// call parsedPomFiles should be nill. It is used to prevent cycles (endless loops). +// call parsedPomFiles should be nil. It is used to prevent cycles (endless loops). func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) (string, map[string]string) { + pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]", findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version) @@ -138,19 +176,23 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { // Nothing new here, already parsed - log.Info("Nothing new here, already processed.") - return "", nil + log.Info("2 Nothing new here, already processed.") + return "" } else { parsedPomFiles[pomCoordinates] = true } - - // Map with all properties defined in all parsed pom files - var allProperties map[string]string = make(map[string]string) - addMissingPropertiesFromProject(allProperties, pom) - // If a parent exists, first parse the parent POM. It may contain required properties and/or - // managed dependencies. + foundVersion := "" + if pom.DependencyManagement != nil { + foundVersion = findVersionInDependencyManagement( + ctx, findGroupID, findArtifactID, pom, cfg, allProperties, parsedPomFiles) + } + if isPropertyResolved(foundVersion) { + return foundVersion + } + + // If a parent exists, search it recursively. if pom.Parent != nil { parentGroupID := *pom.Parent.GroupID parentArtifactID := *pom.Parent.ArtifactID @@ -160,63 +202,22 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup if parentPom != nil { log.Infof("Found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) - // Mark this parent pom as parsed to prevent re-parsing/cycles later on - parsedPomFilesCache[MavenCoordinate{*parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version}] = parentPom - addMissingPropertiesFromProject(allProperties, parentPom) addPropertiesToProject(parentPom, allProperties) - - // TODO: Recurse into parent to gather properties (searching dependencies of parent poms) - // TODO: create function to just gather properties from all parents - // _, parentProperties := recursivelyFindVersionFromManagedOrInherited( - // ctx, findGroupID, findArtifactID, parentPom, cfg, parsedPomFiles) - // log.Info("return 1") - // addMissingPropertiesToMap(allProperties, parentProperties) - // addPropertiesToProject(parentPom, allProperties) - - foundVersion := "" - - if parentPom.Dependencies != nil { - foundVersion = findVersionInDependencies(findGroupID, findArtifactID, parentPom) - } - if foundVersion == "" && parentPom.DependencyManagement != nil { - foundVersion = findVersionInDependencyManagement( - ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, parsedPomFiles) - log.Infof("====> 1 Here: %s", foundVersion) - } - log.Infof("====> 2 Here: %s", foundVersion) - foundVersion = resolveProperty(*parentPom, &foundVersion, "version") - log.Infof("====> 3 Here: %s", foundVersion) - if isPropertyResolved(foundVersion) { - log.Infof("1Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) - return foundVersion, allProperties - } - // if foundVersion != "" || strings.HasPrefix(foundVersion, "${}") { - // foundVersion := resolveProperty(*parentPom, &foundVersion, "version") - // log.Infof("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) - // return foundVersion, allProperties - // } + foundVersion = recursivelyFindVersionFromManagedOrInherited( + ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, parsedPomFiles) } else { log.Warnf("unable to get parent pom [%s, %s, %s]: %v", parentGroupID, parentArtifactID, parentVersion, err) } } - foundVersion := "" - if pom.DependencyManagement != nil { - foundVersion = findVersionInDependencyManagement( - ctx, findGroupID, findArtifactID, pom, cfg, allProperties, parsedPomFiles) - } - if pom.Dependencies != nil { - foundVersion = findVersionInDependencies(findGroupID, findArtifactID, pom) - } - if foundVersion == "" { - log.Infof("No version found for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + log.Infof("No version found for dependency: [%s, %s]", findGroupID, findArtifactID) } else { log.Infof("2Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) } - return foundVersion, allProperties + return foundVersion } // Returns true when value is not empty and does not start with "${" (contains an unresolved property). @@ -234,6 +235,9 @@ func getPomFromMavenOrCache(ctx context.Context, parentGroupID, parentArtifactID parentPom, err = getPomFromMavenRepo(ctx, parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) if err == nil { addPropertiesToProject(parentPom, allProperties) + addMissingPropertiesFromProject(allProperties, parentPom) + // Store in cache + parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] = parentPom } } return parentPom, err @@ -245,25 +249,27 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { for _, dependency := range *getPomManagedDependencies(pom) { - // log.Tracef(" Found managed dependency: [%s, %s, %s]", - // safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) + log.Tracef(" Found managed dependency: [%s, %s, %s]", + safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) // imported pom files should be treated just like parent poms, they are use to define versions of dependencies if dependency.Type != nil && dependency.Scope != nil && *dependency.Type == "pom" && *dependency.Scope == "import" { - bomVersion := resolveProperty(*pom, dependency.Version, "version") + bomVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) log.Debugf("Found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) // Recurse into BOM, which should be treated just like a parent pom bomProject, err := getPomFromMavenOrCache(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) if err == nil { - foundVersion, bomProperties := recursivelyFindVersionFromManagedOrInherited(ctx, findGroupID, findArtifactID, bomProject, cfg, parsedPomFiles) + foundVersion := recursivelyFindVersionFromManagedOrInherited( + ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, parsedPomFiles) + log.Info("return 2") log.Debugf("Finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) - addMissingPropertiesToMap(allProperties, bomProperties) + addMissingPropertiesFromProject(allProperties, pom) + if isPropertyResolved(foundVersion) { - log.Infof("---> HERE!: %s", foundVersion) return foundVersion } if foundVersion != "" { @@ -276,6 +282,9 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { + if strings.HasPrefix(*dependency.Version, "${") { + + } foundVersion := resolveProperty(*pom, dependency.Version, "version") if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { log.Tracef("Found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) @@ -290,9 +299,9 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt // Find given dependency (groupID, artifactID) in the dependencies section of project 'pom'. func findVersionInDependencies(groupID, artifactID string, pom *gopom.Project) string { - for _, dependency := range *pom.Dependencies { + for _, dependency := range *getPomDependencies(pom) { if *dependency.GroupID == groupID && *dependency.ArtifactID == artifactID { - depVersion := resolveProperty(*pom, dependency.Version, "version") + depVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) // TODO: -> trace log.Infof("Found dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, depVersion) return depVersion @@ -303,6 +312,7 @@ func findVersionInDependencies(groupID, artifactID string, pom *gopom.Project) s } func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { + return make([]string, 0) log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", groupID, artifactID, groupID, artifactID, version) var licenses []string diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index aa2f7bf3f3e..c1d51e663f3 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -8,6 +8,7 @@ import ( "io" "reflect" "regexp" + "runtime/debug" "strings" "github.com/saintfish/chardet" @@ -43,13 +44,14 @@ var parsedPomFilesCache map[MavenCoordinate]*gopom.Project = make(map[MavenCoord func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) + if err != nil { return nil, nil, err } var pkgs []pkg.Package - // Add all properties defined in parent poms to this project for resolving properties. + // Add all properties defined in parent poms to this project for resolving properties later on. if pom.Parent != nil { var allProperties map[string]string = make(map[string]string) getPropertiesFromParentPoms( @@ -142,26 +144,27 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende name := safeString(dep.ArtifactID) version := resolveProperty(pom, dep.Version, "version") log.Infof("New dependency: [%s, %s, %s].", groupId, artifactId, version) - var allProperties map[string]string + var allProperties map[string]string = make(map[string]string) + addMissingPropertiesFromProject(allProperties, &pom) licenses := make([]pkg.License, 0) if cfg.UseNetwork { if version == "" { // If we have no version then let's try to get it from a parent pom DependencyManagement section - version, allProperties = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, nil) + version = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, allProperties, nil) log.Info("return 3") - pom.Properties.Entries = allProperties version = resolveProperty(pom, &version, "version") } else if strings.HasPrefix(version, "${") { - log.Error("THIS SHOULD NOT HAPPEN!") + log.Error("===>THIS SHOULD NOT HAPPEN!") // If we are missing the property for this version, search the pom hierarchy for it. if pom.Parent != nil { getPropertiesFromParentPoms(ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg, nil) } - version = resolveProperty(pom, &version, "version") + // version = resolveProperty(pom, &version, "version") + version = resolveProperty(pom, &version, getPropertyName(version)) } - if version != "" { + if isPropertyResolved(version) { parentLicenses := recursivelyFindLicensesFromParentPom( ctx, @@ -181,7 +184,7 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende } if strings.HasPrefix(version, "${") { - log.Warnf("Got version '%s' for artifact: %s", version, name) + log.Warnf("Got unresolved version '%s' for artifact: %s", version, name) } p := pkg.Package{ @@ -233,7 +236,8 @@ func decodePomXML(content io.Reader) (project gopom.Project, err error) { project.Properties.Entries["project.version"] = *project.Version } } - + // Store in cache + parsedPomFilesCache[MavenCoordinate{*project.GroupID, *project.ArtifactID, *project.Version}] = &project return project, nil } @@ -302,25 +306,30 @@ func cleanDescription(original *string) (cleaned string) { // If no match is found, the entire expression including ${} is returned // //nolint:gocognit -func resolveProperty(pom gopom.Project, property *string, propertyName string) string { - propertyCase := safeString(property) +func resolveProperty(pom gopom.Project, propertyValue *string, propertyName string) string { + propertyCase := safeString(propertyValue) if !strings.Contains(propertyCase, "${") { //nothing to resolve // log.Tracef("resolving property: value [%s] contains no variable", propertyName) return propertyCase } + if propertyCase == "${mockito-junit-jupiter.version}" { + debug.PrintStack() + log.Debugf("allProperties: %+v", pom.Properties.Entries) + } log.WithFields("existingPropertyValue", propertyCase, "propertyName", propertyName).Trace("resolving property") return propertyMatcher.ReplaceAllStringFunc(propertyCase, func(match string) string { - propertyName := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } entries := pomProperties(pom) - if value, ok := entries[propertyName]; ok { - log.WithFields("existingPropertyValue", value, "propertyName", propertyName).Trace("resolved property") + value := resolveRecursiveByPropertyName(entries, match) + if isPropertyResolved(value) { + log.WithFields("PropertyValue", value, "propertyName", match).Trace("resolved property") return value } // if we don't find anything directly in the pom properties, // see if we have a project.x expression and process this based // on the xml tags in gopom + propertyName := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } parts := strings.Split(propertyName, ".") numParts := len(parts) if numParts > 1 && strings.TrimSpace(parts[0]) == "project" { From eb3f72dcb05f1581906c6ba775aacbdc7d79f6cf Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:13:40 +0100 Subject: [PATCH 05/42] Property resolution working Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/archive_parser.go | 20 ++-- .../pkg/cataloger/java/archive_parser_test.go | 7 +- syft/pkg/cataloger/java/maven_repo_utils.go | 96 ++++++++++--------- syft/pkg/cataloger/java/parse_pom_xml.go | 30 ++++-- syft/pkg/cataloger/java/parse_pom_xml_test.go | 4 +- 5 files changed, 92 insertions(+), 65 deletions(-) diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 8a203749080..3f98068d265 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -67,7 +67,7 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f if err != nil { return nil, nil, err } - return parser.parse(ctx) + return parser.parse(ctx, gap.cfg) } // uniquePkgKey creates a unique string to identify the given package. @@ -107,7 +107,7 @@ func newJavaArchiveParser(reader file.LocationReadCloser, detectNested bool, cfg } // parse the loaded archive and return all packages found. -func (j *archiveParser) parse(ctx context.Context) ([]pkg.Package, []artifact.Relationship, error) { +func (j *archiveParser) parse(ctx context.Context, cfg ArchiveCatalogerConfig) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package var relationships []artifact.Relationship @@ -119,7 +119,7 @@ func (j *archiveParser) parse(ctx context.Context) ([]pkg.Package, []artifact.Re // find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg // NOTE: we cannot generate sha1 digests from packages discovered via pom.properties/pom.xml - auxPkgs, err := j.discoverPkgsFromAllMavenFiles(ctx, parentPkg) + auxPkgs, err := j.discoverPkgsFromAllMavenFiles(ctx, parentPkg, cfg) if err != nil { return nil, nil, err } @@ -221,7 +221,7 @@ func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaMan 3. manifest 4. filename */ - name, version, pomLicenses := j.guessMainPackageNameAndVersionFromPomInfo(ctx) + name, version, pomLicenses := j.guessMainPackageNameAndVersionFromPomInfo(ctx, j.cfg) if name == "" { name = selectName(manifest, j.fileInfo) } @@ -281,7 +281,7 @@ type parsedPomProject struct { Licenses []pkg.License } -func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Context) (name, version string, licenses []pkg.License) { +func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Context, cfg ArchiveCatalogerConfig) (name, version string, licenses []pkg.License) { pomPropertyMatches := j.fileManifest.GlobMatch(false, pomPropertiesGlob) pomMatches := j.fileManifest.GlobMatch(false, pomXMLGlob) var pomPropertiesObject pkg.JavaPomProperties @@ -289,7 +289,7 @@ func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Co // Find the pom.properties/pom.xml if the names seem like a plausible match properties, _ := pomPropertiesByParentPath(j.archivePath, j.location, pomPropertyMatches) - projects, _ := pomProjectByParentPath(j.archivePath, j.location, pomMatches) + projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, pomMatches, cfg) for parentPath, propertiesObj := range properties { if artifactIDMatchesFilename(propertiesObj.ArtifactID, j.fileInfo.name) { @@ -359,7 +359,7 @@ func findPomLicenses(ctx context.Context, pomProjectObject *parsedPomProject, cf // parent package, returning all listed Java packages found for each pom // properties discovered and potentially updating the given parentPkg with new // data. -func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, error) { +func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, parentPkg *pkg.Package, cfg ArchiveCatalogerConfig) ([]pkg.Package, error) { if parentPkg == nil { return nil, nil } @@ -373,7 +373,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren } // pom.xml - projects, err := pomProjectByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob)) + projects, err := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob), cfg) if err != nil { return nil, err } @@ -543,7 +543,7 @@ func pomPropertiesByParentPath(archivePath string, location file.Location, extra return propertiesByParentPath, nil } -func pomProjectByParentPath(archivePath string, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) { +func pomProjectByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string, cfg ArchiveCatalogerConfig) (map[string]*parsedPomProject, error) { contentsOfMavenProjectFiles, err := intFile.ContentsFromZip(archivePath, extractPaths...) if err != nil { return nil, fmt.Errorf("unable to extract maven files: %w", err) @@ -552,7 +552,7 @@ func pomProjectByParentPath(archivePath string, location file.Location, extractP projectByParentPath := make(map[string]*parsedPomProject) for filePath, fileContents := range contentsOfMavenProjectFiles { // TODO: when we support locations of paths within archives we should start passing the specific pom.xml location object instead of the top jar - pomProject, err := parsePomXMLProject(filePath, strings.NewReader(fileContents), location) + pomProject, err := parsePomXMLProject(ctx, filePath, strings.NewReader(fileContents), location, cfg) if err != nil { log.WithFields("contents-path", filePath, "location", location.Path()).Warnf("failed to parse pom.xml: %+v", err) continue diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 4c49d22fb53..f9b00febd00 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -138,7 +138,7 @@ func TestSearchMavenForLicenses(t *testing.T) { defer cleanupFn() // assert licenses are discovered from upstream - _, _, licenses := ap.guessMainPackageNameAndVersionFromPomInfo(context.Background()) + _, _, licenses := ap.guessMainPackageNameAndVersionFromPomInfo(context.Background(), tc.config) assert.Equal(t, tc.expectedLicenses, licenses) }) } @@ -424,14 +424,15 @@ func TestParseJar(t *testing.T) { test.expected[k] = p } + cfg := ArchiveCatalogerConfig{UseNetwork: false} parser, cleanupFn, err := newJavaArchiveParser(file.LocationReadCloser{ Location: file.NewLocation(fixture.Name()), ReadCloser: fixture, - }, false, ArchiveCatalogerConfig{UseNetwork: false}) + }, false, cfg) defer cleanupFn() require.NoError(t, err) - actual, _, err := parser.parse(context.Background()) + actual, _, err := parser.parse(context.Background(), cfg) require.NoError(t, err) if len(actual) != len(test.expected) { diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index c3325b649aa..4637f6f4721 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -81,20 +81,20 @@ func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom } } -// Add all properties from the map 'additionalProperties' to the map 'allProperties' that are not already in the map. -func addMissingPropertiesToMap(allProperties, additionalProperties map[string]string) { - if len(additionalProperties) > 0 { - for name, value := range additionalProperties { - _, exists := additionalProperties[name] - if !exists { - allProperties[name] = value - // log.Tracef(" Added property to allProperties %s=%s", name, value) - } - } - } else { - log.Tracef("addMissingPropertiesToMap: Supplied map was empty.") - } -} +// // Add all properties from the map 'additionalProperties' to the map 'allProperties' that are not already in the map. +// func addMissingPropertiesToMap(allProperties, additionalProperties map[string]string) { +// if len(additionalProperties) > 0 { +// for name, value := range additionalProperties { +// _, exists := additionalProperties[name] +// if !exists { +// allProperties[name] = value +// // log.Tracef(" Added property to allProperties %s=%s", name, value) +// } +// } +// } else { +// log.Tracef("addMissingPropertiesToMap: Supplied map was empty.") +// } +// } // Add all properties from map 'allProperties' to the project 'pom' that are not already defined in the pom. // This increases the chance of the 'resolveProperty' function succeeding. @@ -109,7 +109,7 @@ func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) for name, value := range allProperties { _, exists := pom.Properties.Entries[name] if !exists { - allProperties[name] = value + pom.Properties.Entries[name] = value // log.Tracef(" Added property %s=%s to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) } } @@ -147,6 +147,7 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s if parentPom.Parent != nil { getPropertiesFromParentPoms(ctx, allProperties, *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version, cfg, parsedPomFiles) + log.Debugf("getPropertiesFromParentPoms - allProperties count: %i", len(allProperties)) } } else { log.Error("Got empty parent pom, error: %w") @@ -273,7 +274,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt return foundVersion } if foundVersion != "" { - foundVersion = resolveProperty(*pom, dependency.Version, "version") + foundVersion = resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { log.Tracef("Found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) return foundVersion @@ -282,10 +283,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { - if strings.HasPrefix(*dependency.Version, "${") { - - } - foundVersion := resolveProperty(*pom, dependency.Version, "version") + foundVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { log.Tracef("Found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) return foundVersion @@ -313,29 +311,29 @@ func findVersionInDependencies(groupID, artifactID string, pom *gopom.Project) s func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { return make([]string, 0) - log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", - groupID, artifactID, groupID, artifactID, version) - var licenses []string - // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth - for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { - parentPom, err := getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) - if err != nil { - // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - log.Tracef("unable to get parent pom from Maven repository: %v", err) - return []string{} - } - parentLicenses := parseLicensesFromPom(parentPom) - if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { - licenses = parentLicenses - break - } - - groupID = *parentPom.Parent.GroupID - artifactID = *parentPom.Parent.ArtifactID - version = *parentPom.Parent.Version - } - - return licenses + // log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", + // groupID, artifactID, groupID, artifactID, version) + // var licenses []string + // // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth + // for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { + // parentPom, err := getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) + // if err != nil { + // // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error + // log.Tracef("unable to get parent pom from Maven repository: %v", err) + // return []string{} + // } + // parentLicenses := parseLicensesFromPom(parentPom) + // if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { + // licenses = parentLicenses + // break + // } + + // groupID = *parentPom.Parent.GroupID + // artifactID = *parentPom.Parent.ArtifactID + // version = *parentPom.Parent.Version + // } + + // return licenses } func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { @@ -379,6 +377,18 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) } + // Add all properties defined in parent poms to this project for resolving properties later on. + if pom.Parent != nil { + var allProperties map[string]string = make(map[string]string) + getPropertiesFromParentPoms( + ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, + ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) + + log.Debugf("getPomFromMavenRepo - allProperties count: %i", len(allProperties)) + addPropertiesToProject(&pom, allProperties) + log.Debugf("2 project Properties count: %i", len(pom.Properties.Entries)) + } + return &pom, nil } diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index c1d51e663f3..9b3599d82ff 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -8,7 +8,6 @@ import ( "io" "reflect" "regexp" - "runtime/debug" "strings" "github.com/saintfish/chardet" @@ -56,10 +55,13 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. var allProperties map[string]string = make(map[string]string) getPropertiesFromParentPoms( ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, gap.cfg, nil) + log.Debugf("parserPomXML - allProperties count: %i", len(allProperties)) addPropertiesToProject(&pom, allProperties) + log.Debugf("project Properties count: %i", len(pom.Properties.Entries)) } for _, dep := range *getPomDependencies(&pom) { + log.Debugf("parserPomXML - dependency: [%s, %s, %s]", *dep.GroupID, *dep.ArtifactID) p := newPackageFromPom( ctx, pom, @@ -81,12 +83,24 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. return pkgs, nil, nil } -func parsePomXMLProject(path string, reader io.Reader, location file.Location) (*parsedPomProject, error) { - project, err := decodePomXML(reader) +func parsePomXMLProject(ctx context.Context, path string, reader io.Reader, location file.Location, cfg ArchiveCatalogerConfig) (*parsedPomProject, error) { + pom, err := decodePomXML(reader) if err != nil { return nil, err } - return newPomProject(path, project, location), nil + + // Add all properties defined in parent poms to this project for resolving properties later on. + if pom.Parent != nil { + var allProperties map[string]string = make(map[string]string) + getPropertiesFromParentPoms( + ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg, nil) + + log.Debugf("parsePomXMLProject - allProperties count: %i", len(allProperties)) + addPropertiesToProject(&pom, allProperties) + log.Debugf("2 project Properties count: %i", len(pom.Properties.Entries)) + } + + return newPomProject(path, pom, location), nil } func newPomProject(path string, p gopom.Project, location file.Location) *parsedPomProject { @@ -160,6 +174,9 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende if pom.Parent != nil { getPropertiesFromParentPoms(ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg, nil) + log.Debugf("newPackageFromPom - allProperties count: %i", len(allProperties)) + addPropertiesToProject(&pom, allProperties) + log.Debugf("2 project Properties count: %i", len(pom.Properties.Entries)) } // version = resolveProperty(pom, &version, "version") version = resolveProperty(pom, &version, getPropertyName(version)) @@ -313,10 +330,7 @@ func resolveProperty(pom gopom.Project, propertyValue *string, propertyName stri // log.Tracef("resolving property: value [%s] contains no variable", propertyName) return propertyCase } - if propertyCase == "${mockito-junit-jupiter.version}" { - debug.PrintStack() - log.Debugf("allProperties: %+v", pom.Properties.Entries) - } + log.WithFields("existingPropertyValue", propertyCase, "propertyName", propertyName).Trace("resolving property") return propertyMatcher.ReplaceAllStringFunc(propertyCase, func(match string) string { entries := pomProperties(pom) diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index c845233b125..d865fd0f0d8 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -1,6 +1,7 @@ package java import ( + "context" "encoding/base64" "io" "os" @@ -368,8 +369,9 @@ func Test_parsePomXMLProject(t *testing.T) { t.Run(test.name, func(t *testing.T) { fixture, err := os.Open(test.expected.Path) assert.NoError(t, err) + cfg := ArchiveCatalogerConfig{} - actual, err := parsePomXMLProject(fixture.Name(), fixture, jarLocation) + actual, err := parsePomXMLProject(context.Background(), fixture.Name(), fixture, jarLocation, cfg) assert.NoError(t, err) assert.Equal(t, &test.expected, actual) From 2602b81de5e0afbf8495641f61e96546a522ffc2 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Sun, 17 Mar 2024 18:19:16 +0100 Subject: [PATCH 06/42] Get pom from local repo, recursively gather properties Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 465 +++++++++++--------- syft/pkg/cataloger/java/parse_pom_xml.go | 73 ++- 2 files changed, 281 insertions(+), 257 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 4637f6f4721..8b958468925 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -6,6 +6,8 @@ import ( "io" "net/http" "net/url" + "os" + "path/filepath" "strings" "time" @@ -14,6 +16,13 @@ import ( "github.com/anchore/syft/internal/log" ) +// Map containing all pom.xml files that have been parsed. They are cached because properties might have been added +// and also to prevent downloading multiple times from a remote repository. +var parsedPomFilesCache map[MavenCoordinate]*gopom.Project = make(map[MavenCoordinate]*gopom.Project) + +var checkedForMavenLocalRepo bool = false +var mavenLocalRepoDir string = "" + func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (requestURL string, err error) { // groupID needs to go from maven.org -> maven/org urlPath := strings.Split(groupID, ".") @@ -28,135 +37,6 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque return requestURL, err } -func resolveRecursiveByPropertyName(pomProperties map[string]string, propertyName string) string { - if strings.HasPrefix(propertyName, "${") { - name := getPropertyName(propertyName) - // log.Debugf("--Name: %s", name) - if value, ok := pomProperties[name]; ok { - // log.Debugf("--Value: %s", value) - if strings.HasPrefix(value, "${") { - log.Trace("recurse resolveRecursiveByPropertyName") - return resolveRecursiveByPropertyName(pomProperties, value) - } else { - // log.Debugf("++Value: %s", value) - return value - } - } - // log.Debugf("**Value: %s", propertyName) - return propertyName - } else { - // log.Debugf("&&Value: %s", propertyName) - return propertyName - } -} - -func getPropertyName(value string) string { - propertyName := value - if strings.HasPrefix(propertyName, "${") { - propertyName = strings.TrimSpace(propertyName[2 : len(propertyName)-1]) // remove leading ${ and trailing } - } - return propertyName -} - -// Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. -func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom.Project) { - if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { - for name, value := range pom.Properties.Entries { - _, exists := allProperties[name] - if !exists { - value = resolveProperty(*pom, &value, getPropertyName(value)) - allProperties[name] = value - log.Tracef(" Added from project, property %s=%s to allProperties", name, value) - } - } - // resolve - for name, value := range allProperties { - if strings.HasPrefix(value, "${") { - allProperties[name] = resolveRecursiveByPropertyName(allProperties, value) - } - } - - } else { - log.Tracef("addMissingPropertiesToProject: nothing to do for project: %s", *pom.ArtifactID) - } -} - -// // Add all properties from the map 'additionalProperties' to the map 'allProperties' that are not already in the map. -// func addMissingPropertiesToMap(allProperties, additionalProperties map[string]string) { -// if len(additionalProperties) > 0 { -// for name, value := range additionalProperties { -// _, exists := additionalProperties[name] -// if !exists { -// allProperties[name] = value -// // log.Tracef(" Added property to allProperties %s=%s", name, value) -// } -// } -// } else { -// log.Tracef("addMissingPropertiesToMap: Supplied map was empty.") -// } -// } - -// Add all properties from map 'allProperties' to the project 'pom' that are not already defined in the pom. -// This increases the chance of the 'resolveProperty' function succeeding. -func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) { - - if len(allProperties) > 0 { - if pom.Properties == nil { - var props gopom.Properties - props.Entries = make(map[string]string) - pom.Properties = &props - } - for name, value := range allProperties { - _, exists := pom.Properties.Entries[name] - if !exists { - pom.Properties.Entries[name] = value - // log.Tracef(" Added property %s=%s to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) - } - } - } -} - -// Traverse the parent pom hierarchy and return all found properties. -// To be used for resolving property variables. -func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]string, parentGroupID, parentArtifactID, parentVersion string, - cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) { - log.Debugf("Recursively gathering all properties from pom [%s, %s, %s]", parentGroupID, parentArtifactID, parentVersion) - - // Create map to keep track of parsed pom files and to prevent cycles. - if parsedPomFiles == nil { - parsedPomFiles = make(map[MavenCoordinate]bool) - } - log.Tracef("Recursion depth: %+v", len(parsedPomFiles)) - - pomCoordinates := MavenCoordinate{parentGroupID, parentArtifactID, parentVersion} - _, alreadyParsed := parsedPomFiles[pomCoordinates] - if alreadyParsed { - // Nothing new here, already parsed - log.Info("1 Nothing new here, already processed.") - return - } - - parentPom, err := getPomFromMavenOrCache(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) - - if err == nil { - if parentPom != nil { - parsedPomFiles[pomCoordinates] = true - addMissingPropertiesFromProject(allProperties, parentPom) - - // recurse into another parent pom - if parentPom.Parent != nil { - getPropertiesFromParentPoms(ctx, allProperties, *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version, - cfg, parsedPomFiles) - log.Debugf("getPropertiesFromParentPoms - allProperties count: %i", len(allProperties)) - } - } else { - log.Error("Got empty parent pom, error: %w") - } - } else { - log.Errorf("Could not get parent pom: %w", err) - } -} - // Try to find the version of a dependency (groupID, artifactID) by parsing all parent poms and imported managed dependencies (maven BOMs). // Properties are gathered in the order that they are encountered: in Maven the latest definition of a property (highest in hierarchy) is used. // parsedPomFiles contains all previously parsed pom files encountered by earlier invocations of this function on the stack. So for the first @@ -164,20 +44,17 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { - log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]", - findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version) - // Create map to keep track of parsed pom files and to prevent cycles. if parsedPomFiles == nil { parsedPomFiles = make(map[MavenCoordinate]bool) } - log.Tracef("Recursion depth: %+v", len(parsedPomFiles)) + log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %s", + findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version, len(parsedPomFiles)) pomCoordinates := MavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { - // Nothing new here, already parsed - log.Info("2 Nothing new here, already processed.") + log.Debug("Skipping already processed pom.") return "" } else { parsedPomFiles[pomCoordinates] = true @@ -199,10 +76,10 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup parentArtifactID := *pom.Parent.ArtifactID parentVersion := *pom.Parent.Version - parentPom, err := getPomFromMavenOrCache(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) + parentPom, err := getPomFromCacheOrMaven(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) if parentPom != nil { - log.Infof("Found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) + log.Debugf("Found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) addMissingPropertiesFromProject(allProperties, parentPom) addPropertiesToProject(parentPom, allProperties) foundVersion = recursivelyFindVersionFromManagedOrInherited( @@ -214,9 +91,9 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup } if foundVersion == "" { - log.Infof("No version found for dependency: [%s, %s]", findGroupID, findArtifactID) + log.Tracef("No version found for dependency: [%s, %s]", findGroupID, findArtifactID) } else { - log.Infof("2Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + log.Debugf("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) } return foundVersion } @@ -226,47 +103,27 @@ func isPropertyResolved(value string) bool { return value != "" && !strings.HasPrefix(value, "${}") } -// Get a parent pom from cache or download from a Maven repository -func getPomFromMavenOrCache(ctx context.Context, parentGroupID, parentArtifactID, parentVersion string, allProperties map[string]string, - cfg ArchiveCatalogerConfig) (*gopom.Project, error) { - var err error = nil - parentPom, found := parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] - - if !found && cfg.UseNetwork { - parentPom, err = getPomFromMavenRepo(ctx, parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) - if err == nil { - addPropertiesToProject(parentPom, allProperties) - addMissingPropertiesFromProject(allProperties, parentPom) - // Store in cache - parsedPomFilesCache[MavenCoordinate{parentGroupID, parentArtifactID, parentVersion}] = parentPom - } - } - return parentPom, err -} - // Find given dependency (groupID, artifactID) in the dependencyManagement section of project 'pom'. // May recursively call recursivelyFindVersionFromManagedOrInherited when a Maven BOM is found. func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { for _, dependency := range *getPomManagedDependencies(pom) { - log.Tracef(" Found managed dependency: [%s, %s, %s]", + log.Tracef("Got managed dependency: [%s, %s, %s]", safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) // imported pom files should be treated just like parent poms, they are use to define versions of dependencies - if dependency.Type != nil && dependency.Scope != nil && - *dependency.Type == "pom" && *dependency.Scope == "import" { + if safeString(dependency.Type) == "pom" && safeString(dependency.Scope) == "import" { bomVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) log.Debugf("Found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) // Recurse into BOM, which should be treated just like a parent pom - bomProject, err := getPomFromMavenOrCache(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) + bomProject, err := getPomFromCacheOrMaven(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) if err == nil { foundVersion := recursivelyFindVersionFromManagedOrInherited( ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, parsedPomFiles) - log.Info("return 2") - log.Debugf("Finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) + log.Tracef("Finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) addMissingPropertiesFromProject(allProperties, pom) @@ -276,7 +133,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt if foundVersion != "" { foundVersion = resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { - log.Tracef("Found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) + log.Debugf("Found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) return foundVersion } } @@ -285,7 +142,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { foundVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { - log.Tracef("Found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) + log.Debugf("Found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) return foundVersion } } @@ -294,48 +151,131 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt return "" } -// Find given dependency (groupID, artifactID) in the dependencies section of project 'pom'. -func findVersionInDependencies(groupID, artifactID string, pom *gopom.Project) string { +func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { + log.Debugf("=== recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", + groupID, artifactID, groupID, artifactID, version) + var licenses []string + // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth + for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { + parentPom, err := getPomFromCacheOrMaven(ctx, groupID, artifactID, version, make(map[string]string), cfg) + if err != nil { + // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error + log.Tracef("unable to get parent pom from Maven repository: %v", err) + return []string{} + } + parentLicenses := parseLicensesFromPom(parentPom) + if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { + licenses = parentLicenses + break + } + + groupID = *parentPom.Parent.GroupID + artifactID = *parentPom.Parent.ArtifactID + version = *parentPom.Parent.Version + } + + return licenses +} + +// Get a parent pom from cache, local repository or download from a Maven repository +func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version string, allProperties map[string]string, + cfg ArchiveCatalogerConfig) (*gopom.Project, error) { + var err error = nil + + // Try get from cache first. + parentPom, found := parsedPomFilesCache[MavenCoordinate{groupID, artifactID, version}] + + if found { + return parentPom, err + } else { + // Then try to get from local file system. + parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version) + + if !found && cfg.UseNetwork { + // If all fails, then try to get from Maven repository over HTTP + parentPom, err = getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) + if err != nil && parentPom != nil { + found = true + } + } + + if found { + // Get and add all properties defined in parent poms to this project for resolving properties later on. + if parentPom.Parent != nil { + log.Infof("parentPom.Parent = %+v", parentPom.Parent) + getPropertiesFromParentPoms( + ctx, allProperties, *parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version, + ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) + } + addPropertiesToProject(parentPom, allProperties) + addMissingPropertiesFromProject(allProperties, parentPom) + + // Store in cache + parsedPomFilesCache[MavenCoordinate{groupID, artifactID, version}] = parentPom + } + } + return parentPom, err +} + +// Try to get the Pom from the users local repository in the users home dir. +// Returns (nil, false) when file cannot be found or read for any reason. +func getPomFromMavenUserLocalRepository(groupID, artifactID, version string) (*gopom.Project, bool) { + localRepoDir, exists := getLocalRepositoryExists() + + if !exists { + return nil, false + } + + groupPath := filepath.Join(strings.Split(groupID, ".")...) + pomFile := filepath.Join(localRepoDir, groupPath, artifactID, version, artifactID+"-"+version+".pom") - for _, dependency := range *getPomDependencies(pom) { - if *dependency.GroupID == groupID && *dependency.ArtifactID == artifactID { - depVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - // TODO: -> trace - log.Infof("Found dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, depVersion) - return depVersion + if _, err := os.Stat(pomFile); !os.IsNotExist(err) { + log.Debugf("Found pom file: %s", pomFile) + bytes, err := os.ReadFile(pomFile) + if err != nil { + log.Errorf("Could not read pom file: [%s], error: %w", pomFile, err) + return nil, false + } + pom, err := decodePomXML(strings.NewReader(string(bytes))) + if err != nil { + log.Errorf("Could not parse pom file: [%s], error: %w", pomFile, err) + return nil, false } + return &pom, true + } else { + log.Debugf("Could not find pom file: [%s]", pomFile) } - log.Tracef("Dependency not found in dependencies") - return "" + return nil, false } -func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { - return make([]string, 0) - // log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", - // groupID, artifactID, groupID, artifactID, version) - // var licenses []string - // // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth - // for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { - // parentPom, err := getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) - // if err != nil { - // // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - // log.Tracef("unable to get parent pom from Maven repository: %v", err) - // return []string{} - // } - // parentLicenses := parseLicensesFromPom(parentPom) - // if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { - // licenses = parentLicenses - // break - // } - - // groupID = *parentPom.Parent.GroupID - // artifactID = *parentPom.Parent.ArtifactID - // version = *parentPom.Parent.Version - // } - - // return licenses +// Get Maven local repository of current user, if it exists. Only checks once and store the result in 'mavenLocalRepoDir'. +func getLocalRepositoryExists() (string, bool) { + found := false + if checkedForMavenLocalRepo { + if mavenLocalRepoDir != "" { + found = true + } + return mavenLocalRepoDir, found + } else { + if !checkedForMavenLocalRepo { + homeDir, err := os.UserHomeDir() + if err != nil { + log.Errorf("Could not find user home dir: %w", err) + } + localRepoDir := filepath.Join(homeDir, ".m2", "repository") + if _, err := os.Stat(homeDir); !os.IsNotExist(err) { + mavenLocalRepoDir = localRepoDir + found = true + } else { + log.Infof("Local Maven repository not found at [%s],", localRepoDir) + } + checkedForMavenLocalRepo = true + } + return mavenLocalRepoDir, found + } } +// Download the pom file from a (remote) Maven repository over HTTP. func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { if len(groupID) == 0 || len(artifactID) == 0 || len(version) == 0 || strings.HasPrefix(version, "${") { return nil, fmt.Errorf("missing/incomplete maven artifact coordinates: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) @@ -361,6 +301,9 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave if err != nil { return nil, fmt.Errorf("unable to get pom from Maven repository: %w", err) } + if resp.StatusCode == 404 { + return nil, fmt.Errorf("pom not found in Maven repository") + } defer func() { if err := resp.Body.Close(); err != nil { log.Errorf("unable to close body: %+v", err) @@ -377,18 +320,6 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) } - // Add all properties defined in parent poms to this project for resolving properties later on. - if pom.Parent != nil { - var allProperties map[string]string = make(map[string]string) - getPropertiesFromParentPoms( - ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, - ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) - - log.Debugf("getPomFromMavenRepo - allProperties count: %i", len(allProperties)) - addPropertiesToProject(&pom, allProperties) - log.Debugf("2 project Properties count: %i", len(pom.Properties.Entries)) - } - return &pom, nil } @@ -456,3 +387,113 @@ func getPomManagedDependencies(pom *gopom.Project) *[]gopom.Dependency { } } } + +// Traverse the parent pom hierarchy and return all found properties. +// To be used for resolving property references later on while determining versions. +// This function recursively processes each encountered parent pom until no parent pom +// is found. +func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]string, parentGroupID, parentArtifactID, parentVersion string, + cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) { + + // Create map to keep track of parsed pom files and to prevent cycles. + if parsedPomFiles == nil { + parsedPomFiles = make(map[MavenCoordinate]bool) + } + log.Debugf("Recursively gathering all properties from pom [%s, %s, %s], recursion depth: %d", + parentGroupID, parentArtifactID, parentVersion, len(parsedPomFiles)) + + pomCoordinates := MavenCoordinate{parentGroupID, parentArtifactID, parentVersion} + _, alreadyParsed := parsedPomFiles[pomCoordinates] + if alreadyParsed { + // Nothing new here, already parsed + log.Debug("Skipping already processed pom.") + return + } + + parentPom, err := getPomFromCacheOrMaven(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) + + if err == nil { + if parentPom != nil { + parsedPomFiles[pomCoordinates] = true + addMissingPropertiesFromProject(allProperties, parentPom) + + // recurse into another parent pom + if parentPom.Parent != nil { + getPropertiesFromParentPoms(ctx, allProperties, *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version, + cfg, parsedPomFiles) + } + } else { + log.Error("Got empty parent pom, error: %w") + } + } else { + log.Errorf("Could not get parent pom: %w", err) + } +} + +// Resolve references to properties (e.g. '${prop.name}') recursively by searching entries in 'allProperties'. +func resolveRecursiveByPropertyName(pomProperties map[string]string, propertyName string) string { + if strings.HasPrefix(propertyName, "${") { + name := getPropertyName(propertyName) + if value, ok := pomProperties[name]; ok { + if strings.HasPrefix(value, "${") { + return resolveRecursiveByPropertyName(pomProperties, value) + } else { + return value + } + } + } + return propertyName +} + +// If 'value' is a property reference (e.g. '${prop.name}'), return the property name (e.g. prop.name). +// Otherwise return the given 'value' +func getPropertyName(value string) string { + propertyName := value + if strings.HasPrefix(propertyName, "${") { + propertyName = strings.TrimSpace(propertyName[2 : len(propertyName)-1]) // remove leading ${ and trailing } + } + return propertyName +} + +// Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. +func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom.Project) { + if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { + for name, value := range pom.Properties.Entries { + // Add property from pom that is not yet in allProperties map. + _, exists := allProperties[name] + if !exists { + value = resolveProperty(*pom, &value, getPropertyName(value)) + allProperties[name] = value + // log.Tracef("Added property ['%s'='%s'] from pom [%s, %s, %s] to allProperties", name, value, + // *pom.GroupID, *pom.ArtifactID, *pom.Version) + } + } + // Try to resolve any added properties containing property references. + for name, value := range allProperties { + if strings.HasPrefix(value, "${") { + allProperties[name] = resolveRecursiveByPropertyName(allProperties, value) + } + } + + } +} + +// Add all properties from map 'allProperties' to the project 'pom' that are not already defined in the pom. +// This increases the chance of the 'resolveProperty' function succeeding. +func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) { + + if len(allProperties) > 0 { + if pom.Properties == nil { + var props gopom.Properties + props.Entries = make(map[string]string) + pom.Properties = &props + } + for name, value := range allProperties { + _, exists := pom.Properties.Entries[name] + if !exists { + pom.Properties.Entries[name] = value + // log.Tracef("Added property ['%s'='%s'] to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) + } + } + } +} diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 9b3599d82ff..8e9d61af7c2 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -38,12 +38,8 @@ const pomXMLGlob = "*pom.xml" var propertyMatcher = regexp.MustCompile("[$][{][^}]+[}]") -// Map containing all pom.xml files that have been parsed. Used for caching as pom files (might) have been downloaded from the internet -var parsedPomFilesCache map[MavenCoordinate]*gopom.Project = make(map[MavenCoordinate]*gopom.Project) - func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) - if err != nil { return nil, nil, err } @@ -55,13 +51,11 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. var allProperties map[string]string = make(map[string]string) getPropertiesFromParentPoms( ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, gap.cfg, nil) - log.Debugf("parserPomXML - allProperties count: %i", len(allProperties)) addPropertiesToProject(&pom, allProperties) - log.Debugf("project Properties count: %i", len(pom.Properties.Entries)) } for _, dep := range *getPomDependencies(&pom) { - log.Debugf("parserPomXML - dependency: [%s, %s, %s]", *dep.GroupID, *dep.ArtifactID) + log.Debugf("Add dependency to SBOM : [%s, %s, %s]", *dep.GroupID, *dep.ArtifactID, safeString(dep.Version)) p := newPackageFromPom( ctx, pom, @@ -94,10 +88,7 @@ func parsePomXMLProject(ctx context.Context, path string, reader io.Reader, loca var allProperties map[string]string = make(map[string]string) getPropertiesFromParentPoms( ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg, nil) - - log.Debugf("parsePomXMLProject - allProperties count: %i", len(allProperties)) addPropertiesToProject(&pom, allProperties) - log.Debugf("2 project Properties count: %i", len(pom.Properties.Entries)) } return newPomProject(path, pom, location), nil @@ -157,51 +148,43 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende name := safeString(dep.ArtifactID) version := resolveProperty(pom, dep.Version, "version") - log.Infof("New dependency: [%s, %s, %s].", groupId, artifactId, version) var allProperties map[string]string = make(map[string]string) addMissingPropertiesFromProject(allProperties, &pom) licenses := make([]pkg.License, 0) - if cfg.UseNetwork { - if version == "" { - // If we have no version then let's try to get it from a parent pom DependencyManagement section - version = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, allProperties, nil) - log.Info("return 3") - version = resolveProperty(pom, &version, "version") - } else if strings.HasPrefix(version, "${") { - log.Error("===>THIS SHOULD NOT HAPPEN!") - // If we are missing the property for this version, search the pom hierarchy for it. - if pom.Parent != nil { - getPropertiesFromParentPoms(ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, - cfg, nil) - log.Debugf("newPackageFromPom - allProperties count: %i", len(allProperties)) - addPropertiesToProject(&pom, allProperties) - log.Debugf("2 project Properties count: %i", len(pom.Properties.Entries)) - } - // version = resolveProperty(pom, &version, "version") - version = resolveProperty(pom, &version, getPropertyName(version)) + if version == "" { + // If we have no version then let's try to get it from a parent pom DependencyManagement section + version = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, allProperties, nil) + version = resolveProperty(pom, &version, "version") + } else if strings.HasPrefix(version, "${") { + // If we are missing the property for this version, search the pom hierarchy for it. + if pom.Parent != nil { + getPropertiesFromParentPoms(ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, + cfg, nil) + addPropertiesToProject(&pom, allProperties) } - if isPropertyResolved(version) { - - parentLicenses := recursivelyFindLicensesFromParentPom( - ctx, - m.PomProperties.GroupID, - m.PomProperties.ArtifactID, - version, - cfg) - - if len(parentLicenses) > 0 { - for _, licenseName := range parentLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) - } + version = resolveProperty(pom, &version, getPropertyName(version)) + } + if isPropertyResolved(version) { + + parentLicenses := recursivelyFindLicensesFromParentPom( + ctx, + m.PomProperties.GroupID, + m.PomProperties.ArtifactID, + version, + cfg) + + if len(parentLicenses) > 0 { + for _, licenseName := range parentLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } - } else { - log.Warnf("Could not determine version for package: [%s, %s]", groupId, artifactId) } + } else { + log.Warnf("Could not determine version for package: [%s, %s]", groupId, artifactId) } if strings.HasPrefix(version, "${") { - log.Warnf("Got unresolved version '%s' for artifact: %s", version, name) + log.Infof("Got unresolved version '%s' for artifact: %s", version, name) } p := pkg.Package{ From 366876652f2e7a72b26eadc68a5455dcf23ecb28 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:20:56 +0200 Subject: [PATCH 07/42] Use local Maven repository for resolving artifacts Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- cmd/syft/internal/options/catalog.go | 39 ++++++++++- cmd/syft/internal/options/java.go | 1 + cmd/syft/sbom.cyclonedx.build.json | 0 go.mod | 1 + go.sum | 1 + sbom.cyclonedx.build.json | 1 + .../pkg/cataloger/java/archive_parser_test.go | 8 ++- syft/pkg/cataloger/java/config.go | 7 ++ syft/pkg/cataloger/java/maven_repo_utils.go | 35 ++++++---- syft/pkg/cataloger/java/parse_pom_xml.go | 67 ++++++++++++------- syft/pkg/cataloger/java/parse_pom_xml_test.go | 1 + 11 files changed, 122 insertions(+), 39 deletions(-) create mode 100644 cmd/syft/sbom.cyclonedx.build.json create mode 100644 sbom.cyclonedx.build.json diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index f12ea41cc5d..0fbdecba78c 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -5,7 +5,10 @@ import ( "sort" "strings" + "github.com/gookit/color" "github.com/iancoleman/strcase" + "github.com/pborman/indent" + "gopkg.in/yaml.v3" "github.com/anchore/clio" "github.com/anchore/fangs" @@ -71,7 +74,8 @@ func DefaultCatalog() Catalog { } func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { - return syft.DefaultCreateSBOMConfig(). + + config := syft.DefaultCreateSBOMConfig(). WithTool(id.Name, id.Version). WithParallelism(cfg.Parallelism). WithRelationshipsConfig(cfg.ToRelationshipsConfig()). @@ -83,6 +87,38 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithDefaults(cfg.DefaultCatalogers...). WithExpression(cfg.SelectCatalogers...), ) + + // log.Tracef("scan SBOMConfig: %+v", config) + logConfiguration(config.Packages.JavaArchive) + return config +} + +func logConfiguration(cfg java.ArchiveCatalogerConfig) { + var sb strings.Builder + + var str string + // yaml is pretty human friendly (at least when compared to json) + cfgBytes, err := yaml.Marshal(cfg) + if err != nil { + str = fmt.Sprintf("%+v", err) + } else { + str = string(cfgBytes) + } + + str = strings.TrimSpace(str) + + if str != "" && str != "{}" { + sb.WriteString(str + "\n") + } + + content := sb.String() + + if content != "" { + formatted := color.Magenta.Sprint(indent.String(" ", strings.TrimSpace(content))) + log.Debugf("config:\n%+v", formatted) + } else { + log.Debug("config: (none)") + } } func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { @@ -150,6 +186,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { }, JavaArchive: java.DefaultArchiveCatalogerConfig(). WithUseNetwork(cfg.Java.UseNetwork). + WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository). WithMavenBaseURL(cfg.Java.MavenURL). WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth), } diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index 342954068ee..1e1567c32b2 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -2,6 +2,7 @@ package options type javaConfig struct { UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` + UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } diff --git a/cmd/syft/sbom.cyclonedx.build.json b/cmd/syft/sbom.cyclonedx.build.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/go.mod b/go.mod index ed93ef56a6c..94e24ef5a14 100644 --- a/go.mod +++ b/go.mod @@ -229,6 +229,7 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.41.0 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/go.sum b/go.sum index 6f458d07f37..ef31c646ea8 100644 --- a/go.sum +++ b/go.sum @@ -1327,6 +1327,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sbom.cyclonedx.build.json b/sbom.cyclonedx.build.json new file mode 100644 index 00000000000..e8b76edb910 --- /dev/null +++ b/sbom.cyclonedx.build.json @@ -0,0 +1 @@ +{"$schema":"http://cyclonedx.org/schema/bom-1.5.schema.json","bomFormat":"CycloneDX","specVersion":"1.5","serialNumber":"urn:uuid:3c306081-82af-4400-9997-63e7078d7b7c","version":1,"metadata":{"timestamp":"2024-04-04T14:17:14+02:00","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"[not provided]"}]},"component":{"bom-ref":"2e2aaf91e5ee9005","type":"file","name":"/home/gcalis/data/httpcomponents-core"}},"components":[{"bom-ref":"pkg:github/actions/cache@v3?package-id=ea5fbdd0bd953c00","type":"library","name":"actions/cache","version":"v3","cpe":"cpe:2.3:a:actions\\/cache:actions\\/cache:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/cache@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/maven.yml"}]},{"bom-ref":"pkg:github/actions/checkout@v3?package-id=9bf49ea83d8e4721","type":"library","name":"actions/checkout","version":"v3","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/codeql-analysis.yml"}]},{"bom-ref":"pkg:github/actions/checkout@v3?package-id=17715560247b3075","type":"library","name":"actions/checkout","version":"v3","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/depsreview.yaml"}]},{"bom-ref":"pkg:github/actions/checkout@v3?package-id=0de54a2b1c1ae315","type":"library","name":"actions/checkout","version":"v3","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/maven.yml"}]},{"bom-ref":"pkg:github/actions/dependency-review-action@v3?package-id=d102d9940748538d","type":"library","name":"actions/dependency-review-action","version":"v3","cpe":"cpe:2.3:a:actions\\/dependency-review-action:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/dependency-review-action@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency-review-action:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review_action:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review_action:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency-review:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency-review:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/depsreview.yaml"}]},{"bom-ref":"pkg:github/actions/setup-java@v3?package-id=9c23239543436d31","type":"library","name":"actions/setup-java","version":"v3","cpe":"cpe:2.3:a:actions\\/setup-java:actions\\/setup-java:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/setup-java@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup-java:actions\\/setup_java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup_java:actions\\/setup-java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup_java:actions\\/setup_java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup:actions\\/setup-java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup:actions\\/setup_java:v3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/maven.yml"}]},{"bom-ref":"pkg:maven/commons-cli/commons-cli@1.5.0?package-id=010b3a6cfd53fe6d","type":"library","group":"commons-cli","name":"commons-cli","version":"1.5.0","cpe":"cpe:2.3:a:commons-cli:commons-cli:1.5.0:*:*:*:*:*:*:*","purl":"pkg:maven/commons-cli/commons-cli@1.5.0","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons-cli:commons_cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons_cli:commons-cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons_cli:commons_cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons:commons-cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons:commons_cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"commons-cli"},{"name":"syft:metadata:-:groupID","value":"commons-cli"}]},{"bom-ref":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber?package-id=c0f757c783b64a19","type":"library","group":"org.conscrypt","name":"conscrypt-openjdk-uber","cpe":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"conscrypt-openjdk-uber"},{"name":"syft:metadata:-:groupID","value":"org.conscrypt"}]},{"bom-ref":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber?package-id=71901d19daed7d37","type":"library","group":"org.conscrypt","name":"conscrypt-openjdk-uber","cpe":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"conscrypt-openjdk-uber"},{"name":"syft:metadata:-:groupID","value":"org.conscrypt"}]},{"bom-ref":"pkg:github/github/codeql-action@v2?package-id=9579f159130ae8bd#analyze","type":"library","name":"github/codeql-action/analyze","version":"v2","cpe":"cpe:2.3:a:github\\/codeql-action\\/analyze:github\\/codeql-action\\/analyze:v2:*:*:*:*:*:*:*","purl":"pkg:github/github/codeql-action@v2#analyze","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql-action\\/analyze:github\\/codeql_action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/analyze:github\\/codeql-action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/analyze:github\\/codeql_action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql-action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql_action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/codeql-analysis.yml"}]},{"bom-ref":"pkg:github/github/codeql-action@v2?package-id=9dd3e55e255449f9#init","type":"library","name":"github/codeql-action/init","version":"v2","cpe":"cpe:2.3:a:github\\/codeql-action\\/init:github\\/codeql-action\\/init:v2:*:*:*:*:*:*:*","purl":"pkg:github/github/codeql-action@v2#init","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql-action\\/init:github\\/codeql_action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/init:github\\/codeql-action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/init:github\\/codeql_action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql-action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql_action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/codeql-analysis.yml"}]},{"bom-ref":"pkg:maven/org.hamcrest/hamcrest?package-id=8eb194cb2ac4021d","type":"library","group":"org.hamcrest","name":"hamcrest","cpe":"cpe:2.3:a:hamcrest:hamcrest:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.hamcrest/hamcrest","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"hamcrest"},{"name":"syft:metadata:-:groupID","value":"org.hamcrest"}]},{"bom-ref":"pkg:maven/org.hamcrest/hamcrest?package-id=9dcf00765a01f8f0","type":"library","group":"org.hamcrest","name":"hamcrest","cpe":"cpe:2.3:a:hamcrest:hamcrest:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.hamcrest/hamcrest","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"hamcrest"},{"name":"syft:metadata:-:groupID","value":"org.hamcrest"}]},{"bom-ref":"pkg:maven/org.hamcrest/hamcrest?package-id=a6bc2f001edcdf55","type":"library","group":"org.hamcrest","name":"hamcrest","cpe":"cpe:2.3:a:hamcrest:hamcrest:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.hamcrest/hamcrest","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"hamcrest"},{"name":"syft:metadata:-:groupID","value":"org.hamcrest"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5?package-id=06f452ca662df3d1","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","cpe":"cpe:2.3:a:apache:httpcore5:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5?package-id=09c4a0e9860a560a","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","cpe":"cpe:2.3:a:apache:httpcore5:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5?package-id=ac453cdfa50cbd06","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","cpe":"cpe:2.3:a:apache:httpcore5:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4?package-id=61d83365dff554f1","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"1a909879fee9bad815223190967d9199de79297a"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/target/httpcore5-5.2.4-tests.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5/target/httpcore5-5.2.4-tests.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4?package-id=2b3b8d3364a0dbbb","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"3af020c5a3295abecfeb9f4a078145b883b6874b"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/target/httpcore5-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5/target/httpcore5-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2?package-id=a279be19b9d88ebc","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-h2","cpe":"cpe:2.3:a:apache:httpcore5-h2:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_h2:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-h2"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2@5.2.4?package-id=d796643e389f2e06","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-h2","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5-h2:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"8738d6dcdf91e66ab1f5f29611bab891442c1334"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_h2:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/target/httpcore5-h2-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-h2"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5-h2/target/httpcore5-h2-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive?package-id=6a13ea19533712fb","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-reactive","cpe":"cpe:2.3:a:apache:httpcore5-reactive:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_reactive:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-reactive"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive@5.2.4?package-id=7611b23e585eea6b","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-reactive","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5-reactive:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"1e1af4da412b6d0642023da1e27e4cd6f7258543"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_reactive:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/target/httpcore5-reactive-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-reactive"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5-reactive/target/httpcore5-reactive-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-testing@5.2.4?package-id=4494f8b4fb836d3d","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-testing","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5-testing:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-testing@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"cb560b4e3b193fcb22b595a4264a32eda10e7e04"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_testing:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/target/httpcore5-testing-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-testing"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5-testing/target/httpcore5-testing-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=54fa850ed3b0a738","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=90b7ee5db4cdf400","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=87befb3f5a3892c7","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=dedea66dffcbb2ef","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-core?package-id=2942099118a1605c","type":"library","group":"org.apache.logging.log4j","name":"log4j-core","cpe":"cpe:2.3:a:apache:log4j-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-core"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-core?package-id=dc37187bca6ec8ba","type":"library","group":"org.apache.logging.log4j","name":"log4j-core","cpe":"cpe:2.3:a:apache:log4j-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-core"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-core?package-id=0392d5abcc91b4a7","type":"library","group":"org.apache.logging.log4j","name":"log4j-core","cpe":"cpe:2.3:a:apache:log4j-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-core"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl?package-id=d369582b74e54c18","type":"library","group":"org.apache.logging.log4j","name":"log4j-slf4j-impl","cpe":"cpe:2.3:a:apache:log4j-slf4j-impl:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_slf4j_impl:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-slf4j-impl"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl?package-id=c9a788cb04f175b3","type":"library","group":"org.apache.logging.log4j","name":"log4j-slf4j-impl","cpe":"cpe:2.3:a:apache:log4j-slf4j-impl:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_slf4j_impl:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-slf4j-impl"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl?package-id=cf1e8c5d0c3675ee","type":"library","group":"org.apache.logging.log4j","name":"log4j-slf4j-impl","cpe":"cpe:2.3:a:apache:log4j-slf4j-impl:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_slf4j_impl:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-slf4j-impl"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.mockito/mockito-core?package-id=ffb40ca9d59d7ee7","type":"library","group":"org.mockito","name":"mockito-core","cpe":"cpe:2.3:a:mockito-core:mockito-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.mockito/mockito-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito-core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"mockito-core"},{"name":"syft:metadata:-:groupID","value":"org.mockito"}]},{"bom-ref":"pkg:maven/org.mockito/mockito-core?package-id=b8a0324ff25c7d0f","type":"library","group":"org.mockito","name":"mockito-core","cpe":"cpe:2.3:a:mockito-core:mockito-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.mockito/mockito-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito-core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"mockito-core"},{"name":"syft:metadata:-:groupID","value":"org.mockito"}]},{"bom-ref":"pkg:maven/org.mockito/mockito-core?package-id=d6bf1fd483f274a5","type":"library","group":"org.mockito","name":"mockito-core","cpe":"cpe:2.3:a:mockito-core:mockito-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.mockito/mockito-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito-core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"mockito-core"},{"name":"syft:metadata:-:groupID","value":"org.mockito"}]},{"bom-ref":"pkg:maven/org.reactivestreams/reactive-streams@1.0.4?package-id=9635bc40f8426e4f","type":"library","group":"org.reactivestreams","name":"reactive-streams","version":"1.0.4","cpe":"cpe:2.3:a:reactive-streams:reactive-streams:1.0.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.reactivestreams/reactive-streams@1.0.4","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive-streams:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive_streams:reactive-streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive_streams:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactivestreams:reactive-streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactivestreams:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive:reactive-streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"reactive-streams"},{"name":"syft:metadata:-:groupID","value":"org.reactivestreams"}]},{"bom-ref":"pkg:maven/io.reactivex.rxjava2/rxjava@${rxjava.version}?package-id=a39e0831b99e6769","type":"library","group":"io.reactivex.rxjava2","name":"rxjava","version":"${rxjava.version}","cpe":"cpe:2.3:a:reactivex:rxjava:\\$\\{rxjava.version\\}:*:*:*:*:*:*:*","purl":"pkg:maven/io.reactivex.rxjava2/rxjava@${rxjava.version}","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava2:rxjava:\\$\\{rxjava.version\\}:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava:rxjava:\\$\\{rxjava.version\\}:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"rxjava"},{"name":"syft:metadata:-:groupID","value":"io.reactivex.rxjava2"}]},{"bom-ref":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}?package-id=f7333bb2fdb63b9b","type":"library","group":"io.reactivex.rxjava3","name":"rxjava","version":"${rxjava3.version}","cpe":"cpe:2.3:a:reactivex:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*","purl":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava3:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"rxjava"},{"name":"syft:metadata:-:groupID","value":"io.reactivex.rxjava3"}]},{"bom-ref":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}?package-id=948cdeb56f7918be","type":"library","group":"io.reactivex.rxjava3","name":"rxjava","version":"${rxjava3.version}","cpe":"cpe:2.3:a:reactivex:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*","purl":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava3:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"rxjava"},{"name":"syft:metadata:-:groupID","value":"io.reactivex.rxjava3"}]},{"bom-ref":"pkg:maven/org.slf4j/slf4j-api?package-id=0cd64d32e6da4a4e","type":"library","group":"org.slf4j","name":"slf4j-api","cpe":"cpe:2.3:a:slf4j-api:slf4j-api:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.slf4j/slf4j-api","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j-api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"slf4j-api"},{"name":"syft:metadata:-:groupID","value":"org.slf4j"}]},{"bom-ref":"pkg:maven/org.slf4j/slf4j-api?package-id=ce7df01f51b64348","type":"library","group":"org.slf4j","name":"slf4j-api","cpe":"cpe:2.3:a:slf4j-api:slf4j-api:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.slf4j/slf4j-api","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j-api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"slf4j-api"},{"name":"syft:metadata:-:groupID","value":"org.slf4j"}]},{"bom-ref":"pkg:maven/org.slf4j/slf4j-api?package-id=084e13ec4fa8ecbc","type":"library","group":"org.slf4j","name":"slf4j-api","cpe":"cpe:2.3:a:slf4j-api:slf4j-api:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.slf4j/slf4j-api","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j-api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"slf4j-api"},{"name":"syft:metadata:-:groupID","value":"org.slf4j"}]}]} diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index f9b00febd00..d435e17bb05 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -91,6 +91,7 @@ func TestSearchMavenForLicenses(t *testing.T) { detectNested: false, config: ArchiveCatalogerConfig{ UseNetwork: true, + UseMavenLocalRepository: false, MavenBaseURL: url, MaxParentRecursiveDepth: 2, }, @@ -424,7 +425,10 @@ func TestParseJar(t *testing.T) { test.expected[k] = p } - cfg := ArchiveCatalogerConfig{UseNetwork: false} + cfg := ArchiveCatalogerConfig{ + UseNetwork: false, + UseMavenLocalRepository: false, + } parser, cleanupFn, err := newJavaArchiveParser(file.LocationReadCloser{ Location: file.NewLocation(fixture.Name()), ReadCloser: fixture, @@ -1339,6 +1343,8 @@ func Test_parseJavaArchive_regressions(t *testing.T) { PomProject: &pkg.JavaPomProject{ Path: "META-INF/maven/org.apache.directory.api/api-asn1-api/pom.xml", ArtifactID: "api-asn1-api", + GroupID: "org.apache.directory.api", + Version: "2.0.0", Name: "Apache Directory API ASN.1 API", Description: "ASN.1 API", Parent: &pkg.JavaPomParent{ diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 14c31d33426..cdafa7ba4ce 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -7,6 +7,7 @@ const mavenBaseURL = "https://repo1.maven.org/maven2" type ArchiveCatalogerConfig struct { cataloging.ArchiveSearchConfig `yaml:",inline" json:"" mapstructure:",squash"` UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` + UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepositoryk"` MavenBaseURL string `yaml:"maven-base-url" json:"maven-base-url" mapstructure:"maven-base-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } @@ -15,6 +16,7 @@ func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { return ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.DefaultArchiveSearchConfig(), UseNetwork: false, + UseMavenLocalRepository: true, MavenBaseURL: mavenBaseURL, MaxParentRecursiveDepth: 5, } @@ -25,6 +27,11 @@ func (j ArchiveCatalogerConfig) WithUseNetwork(input bool) ArchiveCatalogerConfi return j } +func (j ArchiveCatalogerConfig) WithUseMavenLocalRepository(input bool) ArchiveCatalogerConfig { + j.UseMavenLocalRepository = input + return j +} + func (j ArchiveCatalogerConfig) WithMavenBaseURL(input string) ArchiveCatalogerConfig { if input != "" { j.MavenBaseURL = input diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 8b958468925..7259d29297b 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -16,9 +16,16 @@ import ( "github.com/anchore/syft/internal/log" ) +// mavenCoordinate is the unique identifier for a package in Maven. +type mavenCoordinate struct { + GroupID string + ArtifactID string + Version string +} + // Map containing all pom.xml files that have been parsed. They are cached because properties might have been added // and also to prevent downloading multiple times from a remote repository. -var parsedPomFilesCache map[MavenCoordinate]*gopom.Project = make(map[MavenCoordinate]*gopom.Project) +var parsedPomFilesCache map[mavenCoordinate]*gopom.Project = make(map[mavenCoordinate]*gopom.Project) var checkedForMavenLocalRepo bool = false var mavenLocalRepoDir string = "" @@ -42,16 +49,16 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque // parsedPomFiles contains all previously parsed pom files encountered by earlier invocations of this function on the stack. So for the first // call parsedPomFiles should be nil. It is used to prevent cycles (endless loops). func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { + pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[mavenCoordinate]bool) string { // Create map to keep track of parsed pom files and to prevent cycles. if parsedPomFiles == nil { - parsedPomFiles = make(map[MavenCoordinate]bool) + parsedPomFiles = make(map[mavenCoordinate]bool) } - log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %s", + log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %d", findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version, len(parsedPomFiles)) - pomCoordinates := MavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} + pomCoordinates := mavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { log.Debug("Skipping already processed pom.") @@ -100,13 +107,13 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup // Returns true when value is not empty and does not start with "${" (contains an unresolved property). func isPropertyResolved(value string) bool { - return value != "" && !strings.HasPrefix(value, "${}") + return value != "" && !strings.HasPrefix(value, "${") } // Find given dependency (groupID, artifactID) in the dependencyManagement section of project 'pom'. // May recursively call recursivelyFindVersionFromManagedOrInherited when a Maven BOM is found. func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[MavenCoordinate]bool) string { + pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[mavenCoordinate]bool) string { for _, dependency := range *getPomManagedDependencies(pom) { log.Tracef("Got managed dependency: [%s, %s, %s]", @@ -183,13 +190,15 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st var err error = nil // Try get from cache first. - parentPom, found := parsedPomFilesCache[MavenCoordinate{groupID, artifactID, version}] + parentPom, found := parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] if found { return parentPom, err } else { // Then try to get from local file system. - parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version) + if cfg.UseMavenLocalRepository { + parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version) + } if !found && cfg.UseNetwork { // If all fails, then try to get from Maven repository over HTTP @@ -211,7 +220,7 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st addMissingPropertiesFromProject(allProperties, parentPom) // Store in cache - parsedPomFilesCache[MavenCoordinate{groupID, artifactID, version}] = parentPom + parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] = parentPom } } return parentPom, err @@ -393,16 +402,16 @@ func getPomManagedDependencies(pom *gopom.Project) *[]gopom.Dependency { // This function recursively processes each encountered parent pom until no parent pom // is found. func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]string, parentGroupID, parentArtifactID, parentVersion string, - cfg ArchiveCatalogerConfig, parsedPomFiles map[MavenCoordinate]bool) { + cfg ArchiveCatalogerConfig, parsedPomFiles map[mavenCoordinate]bool) { // Create map to keep track of parsed pom files and to prevent cycles. if parsedPomFiles == nil { - parsedPomFiles = make(map[MavenCoordinate]bool) + parsedPomFiles = make(map[mavenCoordinate]bool) } log.Debugf("Recursively gathering all properties from pom [%s, %s, %s], recursion depth: %d", parentGroupID, parentArtifactID, parentVersion, len(parsedPomFiles)) - pomCoordinates := MavenCoordinate{parentGroupID, parentArtifactID, parentVersion} + pomCoordinates := mavenCoordinate{parentGroupID, parentArtifactID, parentVersion} _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { // Nothing new here, already parsed diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 8e9d61af7c2..f8403ddfc87 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -10,9 +10,12 @@ import ( "regexp" "strings" + "github.com/gookit/color" + "github.com/pborman/indent" "github.com/saintfish/chardet" "github.com/vifraa/gopom" "golang.org/x/net/html/charset" + "gopkg.in/yaml.v2" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" @@ -21,24 +24,40 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// MavenCoordinate is the unique identifier for a package in Maven. -type MavenCoordinate struct { - GroupID string - ArtifactID string - Version string -} - -// GroupId and ArtifactId of (managed) dependencies -type GroupArtifact struct { - GroupID string - ArtifactID string -} - const pomXMLGlob = "*pom.xml" var propertyMatcher = regexp.MustCompile("[$][{][^}]+[}]") +func logConfiguration(cfg ArchiveCatalogerConfig) { + var sb strings.Builder + + var str string + // yaml is pretty human friendly (at least when compared to json) + cfgBytes, err := yaml.Marshal(cfg) + if err != nil { + str = fmt.Sprintf("%+v", err) + } else { + str = string(cfgBytes) + } + + str = strings.TrimSpace(str) + + if str != "" && str != "{}" { + sb.WriteString(str + "\n") + } + + content := sb.String() + + if content != "" { + formatted := color.Magenta.Sprint(indent.String(" ", strings.TrimSpace(content))) + log.Debugf("config:\n%+v", formatted) + } else { + log.Debug("config: (none)") + } +} + func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + logConfiguration(gap.cfg) pom, err := decodePomXML(reader) if err != nil { return nil, nil, err @@ -226,18 +245,18 @@ func decodePomXML(content io.Reader) (project gopom.Project, err error) { } // If missing, add maven built-in version property often used in multi-module projects - if project.Version != nil { - if project.Properties == nil { - var props gopom.Properties - props.Entries = make(map[string]string) - props.Entries["project.version"] = *project.Version - project.Properties = &props - } else { - project.Properties.Entries["project.version"] = *project.Version - } - } + // if project.Version != nil { + // if project.Properties == nil { + // var props gopom.Properties + // props.Entries = make(map[string]string) + // props.Entries["project.version"] = *project.Version + // project.Properties = &props + // } else { + // project.Properties.Entries["project.version"] = *project.Version + // } + // } // Store in cache - parsedPomFilesCache[MavenCoordinate{*project.GroupID, *project.ArtifactID, *project.Version}] = &project + parsedPomFilesCache[mavenCoordinate{*project.GroupID, *project.ArtifactID, *project.Version}] = &project return project, nil } diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index d865fd0f0d8..fe216a9165f 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -292,6 +292,7 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { IncludeIndexedArchives: true, IncludeUnindexedArchives: true, }, + UseMavenLocalRepository: false, }) pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) }) From e586b31fa8b365219ee0ae23b13e7ff742f7df8b Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 4 Apr 2024 20:04:01 +0200 Subject: [PATCH 08/42] fix logging of error Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 7259d29297b..82b19356dd7 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -432,7 +432,7 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s cfg, parsedPomFiles) } } else { - log.Error("Got empty parent pom, error: %w") + log.Errorf("Got empty parent pom, error: %w") } } else { log.Errorf("Could not get parent pom: %w", err) From 62684c7c041f9757e5dd07014f523d3dd7288b31 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 4 Apr 2024 20:24:53 +0200 Subject: [PATCH 09/42] fix load default java cataloger config on start Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- cmd/syft/internal/options/catalog.go | 1 + cmd/syft/internal/options/java.go | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 0fbdecba78c..d6b86efb523 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -66,6 +66,7 @@ func DefaultCatalog() Catalog { Package: defaultPackageConfig(), LinuxKernel: defaultLinuxKernelConfig(), Golang: defaultGolangConfig(), + Java: defaultJavaConfig(), File: defaultFileConfig(), Relationships: defaultRelationshipsConfig(), Source: defaultSourceConfig(), diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index 1e1567c32b2..c7e24d45efe 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -1,8 +1,21 @@ package options +import "github.com/anchore/syft/syft/pkg/cataloger/java" + type javaConfig struct { UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } + +func defaultJavaConfig() javaConfig { + def := java.DefaultArchiveCatalogerConfig() + + return javaConfig{ + UseNetwork: def.UseNetwork, + UseMavenLocalRepository: def.UseMavenLocalRepository, + MavenURL: def.MavenBaseURL, + MaxParentRecursiveDepth: def.MaxParentRecursiveDepth, + } +} From 902520bae55c74b56635f456a259150bb5a7be73 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 4 Apr 2024 20:25:16 +0200 Subject: [PATCH 10/42] fix logging of license parsing errors Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/file/license.go | 2 +- syft/pkg/license.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/syft/file/license.go b/syft/file/license.go index 12d487d540c..fe9b5b93f29 100644 --- a/syft/file/license.go +++ b/syft/file/license.go @@ -21,7 +21,7 @@ type LicenseEvidence struct { func NewLicense(value string) License { spdxExpression, err := license.ParseExpression(value) if err != nil { - log.Trace("unable to parse license expression: %s, %w", value, err) + log.Tracef("unable to parse license expression: '%s', error: '%v'", value, err) } return License{ diff --git a/syft/pkg/license.go b/syft/pkg/license.go index 06c35a69ab0..71dbb17b920 100644 --- a/syft/pkg/license.go +++ b/syft/pkg/license.go @@ -70,7 +70,7 @@ func NewLicenseFromType(value string, t license.Type) License { var err error spdxExpression, err = license.ParseExpression(value) if err != nil { - log.Trace("unable to parse license expression: %w", err) + log.Tracef("unable to parse license expression: '%s', error: '%v'", value, err) } } From 78f0c24cdb8c5acfff8476c06c177a052c14aa55 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 4 Apr 2024 20:29:17 +0200 Subject: [PATCH 11/42] cleanup logging: start with lowercase Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 51 ++++++++++----------- syft/pkg/cataloger/java/parse_pom_xml.go | 21 ++------- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 82b19356dd7..8fcd55eeb93 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -55,13 +55,13 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup if parsedPomFiles == nil { parsedPomFiles = make(map[mavenCoordinate]bool) } - log.Debugf("Recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %d", + log.Debugf("recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %d", findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version, len(parsedPomFiles)) pomCoordinates := mavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { - log.Debug("Skipping already processed pom.") + log.Debug("skipping already processed pom.") return "" } else { parsedPomFiles[pomCoordinates] = true @@ -86,7 +86,7 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup parentPom, err := getPomFromCacheOrMaven(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) if parentPom != nil { - log.Debugf("Found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) + log.Debugf("found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) addMissingPropertiesFromProject(allProperties, parentPom) addPropertiesToProject(parentPom, allProperties) foundVersion = recursivelyFindVersionFromManagedOrInherited( @@ -98,9 +98,9 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup } if foundVersion == "" { - log.Tracef("No version found for dependency: [%s, %s]", findGroupID, findArtifactID) + log.Tracef("no version found for dependency: [%s, %s]", findGroupID, findArtifactID) } else { - log.Debugf("Found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) + log.Debugf("found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) } return foundVersion } @@ -116,21 +116,21 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[mavenCoordinate]bool) string { for _, dependency := range *getPomManagedDependencies(pom) { - log.Tracef("Got managed dependency: [%s, %s, %s]", + log.Tracef("got managed dependency: [%s, %s, %s]", safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) // imported pom files should be treated just like parent poms, they are use to define versions of dependencies if safeString(dependency.Type) == "pom" && safeString(dependency.Scope) == "import" { bomVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - log.Debugf("Found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) + log.Debugf("found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) // Recurse into BOM, which should be treated just like a parent pom bomProject, err := getPomFromCacheOrMaven(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) if err == nil { foundVersion := recursivelyFindVersionFromManagedOrInherited( ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, parsedPomFiles) - log.Tracef("Finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) + log.Tracef("finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) addMissingPropertiesFromProject(allProperties, pom) @@ -140,7 +140,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt if foundVersion != "" { foundVersion = resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { - log.Debugf("Found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) + log.Debugf("found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) return foundVersion } } @@ -149,17 +149,17 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { foundVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { - log.Debugf("Found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) + log.Debugf("found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) return foundVersion } } } - log.Tracef("Dependency not found in dependencyManagement") + log.Tracef("dependency not found in dependencyManagement") return "" } func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { - log.Debugf("=== recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", + log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", groupID, artifactID, groupID, artifactID, version) var licenses []string // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth @@ -211,7 +211,6 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st if found { // Get and add all properties defined in parent poms to this project for resolving properties later on. if parentPom.Parent != nil { - log.Infof("parentPom.Parent = %+v", parentPom.Parent) getPropertiesFromParentPoms( ctx, allProperties, *parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version, ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) @@ -239,20 +238,20 @@ func getPomFromMavenUserLocalRepository(groupID, artifactID, version string) (*g pomFile := filepath.Join(localRepoDir, groupPath, artifactID, version, artifactID+"-"+version+".pom") if _, err := os.Stat(pomFile); !os.IsNotExist(err) { - log.Debugf("Found pom file: %s", pomFile) + log.Debugf("found pom file: %s", pomFile) bytes, err := os.ReadFile(pomFile) if err != nil { - log.Errorf("Could not read pom file: [%s], error: %w", pomFile, err) + log.Errorf("could not read pom file: [%s], error: %w", pomFile, err) return nil, false } pom, err := decodePomXML(strings.NewReader(string(bytes))) if err != nil { - log.Errorf("Could not parse pom file: [%s], error: %w", pomFile, err) + log.Errorf("could not parse pom file: [%s], error: %w", pomFile, err) return nil, false } return &pom, true } else { - log.Debugf("Could not find pom file: [%s]", pomFile) + log.Debugf("could not find pom file: [%s]", pomFile) } return nil, false } @@ -269,14 +268,14 @@ func getLocalRepositoryExists() (string, bool) { if !checkedForMavenLocalRepo { homeDir, err := os.UserHomeDir() if err != nil { - log.Errorf("Could not find user home dir: %w", err) + log.Errorf("could not find user home dir: %w", err) } localRepoDir := filepath.Join(homeDir, ".m2", "repository") if _, err := os.Stat(homeDir); !os.IsNotExist(err) { mavenLocalRepoDir = localRepoDir found = true } else { - log.Infof("Local Maven repository not found at [%s],", localRepoDir) + log.Infof("local Maven repository not found at [%s],", localRepoDir) } checkedForMavenLocalRepo = true } @@ -328,7 +327,7 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave if err != nil { return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) } - + log.Tracef("fetched parent pom from Maven repository.") return &pom, nil } @@ -408,14 +407,14 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s if parsedPomFiles == nil { parsedPomFiles = make(map[mavenCoordinate]bool) } - log.Debugf("Recursively gathering all properties from pom [%s, %s, %s], recursion depth: %d", + log.Debugf("recursively gathering all properties from pom [%s, %s, %s], recursion depth: %d", parentGroupID, parentArtifactID, parentVersion, len(parsedPomFiles)) pomCoordinates := mavenCoordinate{parentGroupID, parentArtifactID, parentVersion} _, alreadyParsed := parsedPomFiles[pomCoordinates] if alreadyParsed { // Nothing new here, already parsed - log.Debug("Skipping already processed pom.") + log.Debug("skipping already processed pom.") return } @@ -432,10 +431,10 @@ func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]s cfg, parsedPomFiles) } } else { - log.Errorf("Got empty parent pom, error: %w") + log.Errorf("got empty parent pom, error: %w") } } else { - log.Errorf("Could not get parent pom: %w", err) + log.Errorf("could not get parent pom: %w", err) } } @@ -473,7 +472,7 @@ func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom if !exists { value = resolveProperty(*pom, &value, getPropertyName(value)) allProperties[name] = value - // log.Tracef("Added property ['%s'='%s'] from pom [%s, %s, %s] to allProperties", name, value, + // log.Tracef("added property ['%s'='%s'] from pom [%s, %s, %s] to allProperties", name, value, // *pom.GroupID, *pom.ArtifactID, *pom.Version) } } @@ -501,7 +500,7 @@ func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) _, exists := pom.Properties.Entries[name] if !exists { pom.Properties.Entries[name] = value - // log.Tracef("Added property ['%s'='%s'] to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) + // log.Tracef("added property ['%s'='%s'] to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) } } } diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index f8403ddfc87..7e35411b55e 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -74,7 +74,7 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. } for _, dep := range *getPomDependencies(&pom) { - log.Debugf("Add dependency to SBOM : [%s, %s, %s]", *dep.GroupID, *dep.ArtifactID, safeString(dep.Version)) + log.Debugf("add dependency to SBOM : [%s, %s, %s]", *dep.GroupID, *dep.ArtifactID, safeString(dep.Version)) p := newPackageFromPom( ctx, pom, @@ -89,7 +89,7 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. pkgs = append(pkgs, p) if len(p.Version) == 0 || strings.HasPrefix(p.Version, "${") { - log.Infof("Found artifact without version: %s:%s, version: %q", *dep.GroupID, *dep.ArtifactID, p.Version) + log.Infof("found artifact without version: %s:%s, version: %q", *dep.GroupID, *dep.ArtifactID, p.Version) } } @@ -199,11 +199,11 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende } } } else { - log.Warnf("Could not determine version for package: [%s, %s]", groupId, artifactId) + log.Warnf("could not determine version for package: [%s, %s]", groupId, artifactId) } if strings.HasPrefix(version, "${") { - log.Infof("Got unresolved version '%s' for artifact: %s", version, name) + log.Infof("got unresolved version '%s' for artifact: %s", version, name) } p := pkg.Package{ @@ -244,17 +244,6 @@ func decodePomXML(content io.Reader) (project gopom.Project, err error) { project.Version = project.Parent.Version } - // If missing, add maven built-in version property often used in multi-module projects - // if project.Version != nil { - // if project.Properties == nil { - // var props gopom.Properties - // props.Entries = make(map[string]string) - // props.Entries["project.version"] = *project.Version - // project.Properties = &props - // } else { - // project.Properties.Entries["project.version"] = *project.Version - // } - // } // Store in cache parsedPomFilesCache[mavenCoordinate{*project.GroupID, *project.ArtifactID, *project.Version}] = &project return project, nil @@ -338,7 +327,7 @@ func resolveProperty(pom gopom.Project, propertyValue *string, propertyName stri entries := pomProperties(pom) value := resolveRecursiveByPropertyName(entries, match) if isPropertyResolved(value) { - log.WithFields("PropertyValue", value, "propertyName", match).Trace("resolved property") + log.WithFields("propertyValue", value, "propertyName", match).Trace("resolved property") return value } From 7d4c5a407811dc3a3be27d3a77c424fa1b10c0a0 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Sun, 7 Apr 2024 17:46:08 +0200 Subject: [PATCH 12/42] Make local Maven repository dir configurable Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- cmd/syft/internal/options/catalog.go | 1 + cmd/syft/internal/options/java.go | 2 + syft/pkg/cataloger/java/config.go | 8 ++++ syft/pkg/cataloger/java/maven_repo_utils.go | 47 ++++++++++++++------- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index d6b86efb523..b20b9ee5115 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -188,6 +188,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { JavaArchive: java.DefaultArchiveCatalogerConfig(). WithUseNetwork(cfg.Java.UseNetwork). WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository). + WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir). WithMavenBaseURL(cfg.Java.MavenURL). WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth), } diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index c7e24d45efe..bb9697f055d 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -5,6 +5,7 @@ import "github.com/anchore/syft/syft/pkg/cataloger/java" type javaConfig struct { UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` + MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } @@ -15,6 +16,7 @@ func defaultJavaConfig() javaConfig { return javaConfig{ UseNetwork: def.UseNetwork, UseMavenLocalRepository: def.UseMavenLocalRepository, + MavenLocalRepositoryDir: def.MavenLocalRepositoryDir, MavenURL: def.MavenBaseURL, MaxParentRecursiveDepth: def.MaxParentRecursiveDepth, } diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index cdafa7ba4ce..78bf78c1dde 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -8,15 +8,18 @@ type ArchiveCatalogerConfig struct { cataloging.ArchiveSearchConfig `yaml:",inline" json:"" mapstructure:",squash"` UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepositoryk"` + MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` MavenBaseURL string `yaml:"maven-base-url" json:"maven-base-url" mapstructure:"maven-base-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { + localRepoDir, _ := GetDefaultMavenLocalRepoLocation() return ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.DefaultArchiveSearchConfig(), UseNetwork: false, UseMavenLocalRepository: true, + MavenLocalRepositoryDir: localRepoDir, MavenBaseURL: mavenBaseURL, MaxParentRecursiveDepth: 5, } @@ -32,6 +35,11 @@ func (j ArchiveCatalogerConfig) WithUseMavenLocalRepository(input bool) ArchiveC return j } +func (j ArchiveCatalogerConfig) WithMavenLocalRepositoryDir(input string) ArchiveCatalogerConfig { + j.MavenLocalRepositoryDir = input + return j +} + func (j ArchiveCatalogerConfig) WithMavenBaseURL(input string) ArchiveCatalogerConfig { if input != "" { j.MavenBaseURL = input diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index 8fcd55eeb93..c88e17fcec2 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -124,9 +124,11 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt bomVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) log.Debugf("found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) + // Recurse into BOM, which should be treated just like a parent pom bomProject, err := getPomFromCacheOrMaven(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) - if err == nil { + + if err == nil && bomProject != nil { foundVersion := recursivelyFindVersionFromManagedOrInherited( ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, parsedPomFiles) @@ -139,7 +141,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } if foundVersion != "" { foundVersion = resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { + if isPropertyResolved(foundVersion) { log.Debugf("found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) return foundVersion } @@ -148,7 +150,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { foundVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - if foundVersion != "" && !strings.HasPrefix(foundVersion, "${") { + if isPropertyResolved(foundVersion) { log.Debugf("found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) return foundVersion } @@ -189,6 +191,10 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st cfg ArchiveCatalogerConfig) (*gopom.Project, error) { var err error = nil + if !isPropertyResolved(version) { + return nil, fmt.Errorf("cannot get POM without resolved version: %s", version) + } + // Try get from cache first. parentPom, found := parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] @@ -197,7 +203,7 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st } else { // Then try to get from local file system. if cfg.UseMavenLocalRepository { - parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version) + parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version, cfg) } if !found && cfg.UseNetwork { @@ -227,8 +233,8 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st // Try to get the Pom from the users local repository in the users home dir. // Returns (nil, false) when file cannot be found or read for any reason. -func getPomFromMavenUserLocalRepository(groupID, artifactID, version string) (*gopom.Project, bool) { - localRepoDir, exists := getLocalRepositoryExists() +func getPomFromMavenUserLocalRepository(groupID, artifactID, version string, cfg ArchiveCatalogerConfig) (*gopom.Project, bool) { + localRepoDir, exists := getLocalRepositoryExists(cfg) if !exists { return nil, false @@ -257,7 +263,7 @@ func getPomFromMavenUserLocalRepository(groupID, artifactID, version string) (*g } // Get Maven local repository of current user, if it exists. Only checks once and store the result in 'mavenLocalRepoDir'. -func getLocalRepositoryExists() (string, bool) { +func getLocalRepositoryExists(cfg ArchiveCatalogerConfig) (string, bool) { found := false if checkedForMavenLocalRepo { if mavenLocalRepoDir != "" { @@ -266,16 +272,12 @@ func getLocalRepositoryExists() (string, bool) { return mavenLocalRepoDir, found } else { if !checkedForMavenLocalRepo { - homeDir, err := os.UserHomeDir() - if err != nil { - log.Errorf("could not find user home dir: %w", err) - } - localRepoDir := filepath.Join(homeDir, ".m2", "repository") - if _, err := os.Stat(homeDir); !os.IsNotExist(err) { + localRepoDir := cfg.MavenLocalRepositoryDir + if _, err := os.Stat(localRepoDir); !os.IsNotExist(err) { mavenLocalRepoDir = localRepoDir found = true } else { - log.Infof("local Maven repository not found at [%s],", localRepoDir) + log.Errorf("local Maven repository not found at [%s],", localRepoDir) } checkedForMavenLocalRepo = true } @@ -283,9 +285,24 @@ func getLocalRepositoryExists() (string, bool) { } } +// Get default location of the Maven local repository at /.m2/repository +func GetDefaultMavenLocalRepoLocation() (string, error) { + homeDir, err := os.UserHomeDir() + + if err != nil { + return "", err + } + localRepoDir := filepath.Join(homeDir, ".m2", "repository") + if _, err := os.Stat(homeDir); !os.IsNotExist(err) { + return localRepoDir, nil + } else { + return "", fmt.Errorf("local Maven repository not found at default location [%s],", localRepoDir) + } +} + // Download the pom file from a (remote) Maven repository over HTTP. func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { - if len(groupID) == 0 || len(artifactID) == 0 || len(version) == 0 || strings.HasPrefix(version, "${") { + if len(groupID) == 0 || len(artifactID) == 0 || !isPropertyResolved(version) { return nil, fmt.Errorf("missing/incomplete maven artifact coordinates: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) } requestURL, err := formatMavenPomURL(groupID, artifactID, version, mavenBaseURL) From a77af3743b586bdab4014742651252ae27da2aad Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Sun, 7 Apr 2024 17:46:59 +0200 Subject: [PATCH 13/42] Add unit tests for using remote Maven repo and local Maven repo Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/parse_pom_xml_test.go | 384 ++-- .../commons-parent/54/commons-parent-54.pom | 1988 +++++++++++++++++ .../junit/junit-bom/5.9.0/junit-bom-5.9.0.pom | 159 ++ 3 files changed, 2399 insertions(+), 132 deletions(-) create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index fe216a9165f..61f001cb631 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -137,145 +137,114 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { }{ { input: "test-fixtures/pom/commons-text.pom.xml", - expected: []pkg.Package{ - { - Name: "commons-lang3", - Version: "3.12.0", - PURL: "pkg:maven/org.apache.commons/commons-lang3@3.12.0", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.apache.commons", - ArtifactID: "commons-lang3", - }, - }, - }, - { - Name: "junit-jupiter", - Version: "", - PURL: "pkg:maven/org.junit.jupiter/junit-jupiter", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.junit.jupiter", - ArtifactID: "junit-jupiter", - Scope: "test", - }, - }, - }, - { - Name: "assertj-core", - Version: "3.23.1", - PURL: "pkg:maven/org.assertj/assertj-core@3.23.1", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.assertj", - ArtifactID: "assertj-core", - Scope: "test", - }, - }, - }, - { - Name: "commons-io", - Version: "2.11.0", - PURL: "pkg:maven/commons-io/commons-io@2.11.0", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "commons-io", - ArtifactID: "commons-io", - Scope: "test", - }, - }, - }, - { - Name: "mockito-inline", - Version: "4.8.0", - PURL: "pkg:maven/org.mockito/mockito-inline@4.8.0", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.mockito", - ArtifactID: "mockito-inline", - Scope: "test", - }, - }, + + expected: getCommonsTextExpectedPackages(), + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + for i := range test.expected { + test.expected[i].Locations.Add(file.NewLocation(test.input)) + } + + gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, }, - { - Name: "js", - Version: "22.0.0.2", - PURL: "pkg:maven/org.graalvm.js/js@22.0.0.2", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.graalvm.js", - ArtifactID: "js", - Scope: "test", - }, - }, + UseMavenLocalRepository: false, + }) + pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) + }) + } +} + +func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { + // Using the local repository, the version of junit-jupiter will be resolved + expectedPackages := getCommonsTextExpectedPackages() + + for i := 0; i < len(expectedPackages); i++ { + if expectedPackages[i].Name == "junit-jupiter" { + expPkg := &expectedPackages[i] + expPkg.Version = "5.9.0" + expPkg.PURL = "pkg:maven/org.junit.jupiter/junit-jupiter@5.9.0" + expPkg.Metadata = pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.junit.jupiter", + ArtifactID: "junit-jupiter", + Scope: "test", }, - { - Name: "js-scriptengine", - Version: "22.0.0.2", - PURL: "pkg:maven/org.graalvm.js/js-scriptengine@22.0.0.2", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.graalvm.js", - ArtifactID: "js-scriptengine", - Scope: "test", - }, - }, + } + } + } + + tests := []struct { + input string + expected []pkg.Package + }{ + { + input: "test-fixtures/pom/commons-text.pom.xml", + expected: expectedPackages, + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + for i := range test.expected { + test.expected[i].Locations.Add(file.NewLocation(test.input)) + } + + gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, }, - { - Name: "commons-rng-simple", - Version: "1.4", - PURL: "pkg:maven/org.apache.commons/commons-rng-simple@1.4", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.apache.commons", - ArtifactID: "commons-rng-simple", - Scope: "test", - }, - }, + UseMavenLocalRepository: true, + MavenLocalRepositoryDir: "test-fixtures/pom/maven-repo", + }) + pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) + }) + } +} + +func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { + mux, url, teardown := setup() + defer teardown() + // Using the local repository, the version of junit-jupiter will be resolved + expectedPackages := getCommonsTextExpectedPackages() + + for i := 0; i < len(expectedPackages); i++ { + if expectedPackages[i].Name == "junit-jupiter" { + expPkg := &expectedPackages[i] + expPkg.Version = "5.9.0" + expPkg.PURL = "pkg:maven/org.junit.jupiter/junit-jupiter@5.9.0" + expPkg.Metadata = pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.junit.jupiter", + ArtifactID: "junit-jupiter", + Scope: "test", }, + } + } + } + + tests := []struct { + input string + expected []pkg.Package + requestHandlers []handlerPath + }{ + { + input: "test-fixtures/pom/commons-text.pom.xml", + expected: expectedPackages, + requestHandlers: []handlerPath{ { - Name: "jmh-core", - Version: "1.35", - PURL: "pkg:maven/org.openjdk.jmh/jmh-core@1.35", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.openjdk.jmh", - ArtifactID: "jmh-core", - Scope: "test", - }, - }, + path: "/org/apache/commons/commons-parent/54/commons-parent-54.pom", + handler: generateMockMavenHandler("test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom"), }, { - Name: "jmh-generator-annprocess", - Version: "1.35", - PURL: "pkg:maven/org.openjdk.jmh/jmh-generator-annprocess@1.35", - Language: pkg.Java, - Type: pkg.JavaPkg, - Metadata: pkg.JavaArchive{ - PomProperties: &pkg.JavaPomProperties{ - GroupID: "org.openjdk.jmh", - ArtifactID: "jmh-generator-annprocess", - Scope: "test", - }, - }, + path: "/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom", + handler: generateMockMavenHandler("test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom"), }, }, }, @@ -283,6 +252,11 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { for _, test := range tests { t.Run(test.input, func(t *testing.T) { + // configure maven central requests + for _, hdlr := range test.requestHandlers { + mux.HandleFunc(hdlr.path, hdlr.handler) + } + for i := range test.expected { test.expected[i].Locations.Add(file.NewLocation(test.input)) } @@ -292,6 +266,8 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { IncludeIndexedArchives: true, IncludeUnindexedArchives: true, }, + UseNetwork: true, + MavenBaseURL: url, UseMavenLocalRepository: false, }) pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) @@ -555,3 +531,147 @@ func Test_getUtf8Reader(t *testing.T) { }) } } + +func getCommonsTextExpectedPackages() []pkg.Package { + return []pkg.Package{ + { + Name: "commons-lang3", + Version: "3.12.0", + PURL: "pkg:maven/org.apache.commons/commons-lang3@3.12.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.apache.commons", + ArtifactID: "commons-lang3", + }, + }, + }, + { + Name: "junit-jupiter", + Version: "", + PURL: "pkg:maven/org.junit.jupiter/junit-jupiter", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.junit.jupiter", + ArtifactID: "junit-jupiter", + Scope: "test", + }, + }, + }, + { + Name: "assertj-core", + Version: "3.23.1", + PURL: "pkg:maven/org.assertj/assertj-core@3.23.1", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.assertj", + ArtifactID: "assertj-core", + Scope: "test", + }, + }, + }, + { + Name: "commons-io", + Version: "2.11.0", + PURL: "pkg:maven/commons-io/commons-io@2.11.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "commons-io", + ArtifactID: "commons-io", + Scope: "test", + }, + }, + }, + { + Name: "mockito-inline", + Version: "4.8.0", + PURL: "pkg:maven/org.mockito/mockito-inline@4.8.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.mockito", + ArtifactID: "mockito-inline", + Scope: "test", + }, + }, + }, + { + Name: "js", + Version: "22.0.0.2", + PURL: "pkg:maven/org.graalvm.js/js@22.0.0.2", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.graalvm.js", + ArtifactID: "js", + Scope: "test", + }, + }, + }, + { + Name: "js-scriptengine", + Version: "22.0.0.2", + PURL: "pkg:maven/org.graalvm.js/js-scriptengine@22.0.0.2", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.graalvm.js", + ArtifactID: "js-scriptengine", + Scope: "test", + }, + }, + }, + { + Name: "commons-rng-simple", + Version: "1.4", + PURL: "pkg:maven/org.apache.commons/commons-rng-simple@1.4", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.apache.commons", + ArtifactID: "commons-rng-simple", + Scope: "test", + }, + }, + }, + { + Name: "jmh-core", + Version: "1.35", + PURL: "pkg:maven/org.openjdk.jmh/jmh-core@1.35", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.openjdk.jmh", + ArtifactID: "jmh-core", + Scope: "test", + }, + }, + }, + { + Name: "jmh-generator-annprocess", + Version: "1.35", + PURL: "pkg:maven/org.openjdk.jmh/jmh-generator-annprocess@1.35", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "org.openjdk.jmh", + ArtifactID: "jmh-generator-annprocess", + Scope: "test", + }, + }, + }, + } +} diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom new file mode 100644 index 00000000000..3d4bdb8d499 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom @@ -0,0 +1,1988 @@ + + + + 4.0.0 + + org.apache + apache + 27 + + org.apache.commons + commons-parent + 54 + pom + Apache Commons Parent + The Apache Commons Parent POM provides common settings for all Apache Commons components. + + 2006 + https://commons.apache.org/commons-parent-pom.html + + + + + + 3.3.9 + + + 2022-09-18T13:49:41Z + ${project.version} + RC1 + COMMONSSITE + + 53 + true + Gary Gregory + 86fdc7e2a11262cb + + + + + 1.3 + 1.3 + + + false + + + + + + 1.22 + + 1.0 + 3.4.2 + 3.3.0 + 1.12 + 2.12.1 + 3.2.0 + 9.3 + 2.7 + 3.10.1 + 4.3.0 + EpochMillis + 2.7.1 + + 0.5.5 + 3.0.0-M7 + 5.1.8 + 0.8.8 + 0.16.0 + 3.3.0 + 3.4.1 + 3.3.0 + 3.19.0 + 6.49.0 + 3.4.1 + 0.15 + 1.8.0 + 1.1 + 3.1.0 + 3.0.0 + 6.3.1 + 5.9.0 + + + + 3.12.1 + 3.2.1 + 4.7.2.0 + 4.7.2 + 3.0.0-M7 + 3.0.0-M7 + 3.5.2 + + + ${project.artifactId}-${commons.release.version} + + -bin + ${project.artifactId}-${commons.release.2.version} + + -bin + ${project.artifactId}-${commons.release.3.version} + + -bin + + -bin + + + 1.00 + 0.90 + 0.95 + 0.85 + 0.85 + 0.90 + false + + + ${project.artifactId} + + + ${project.artifactId} + + + org.apache.commons.${commons.packageId} + org.apache.commons.*;version=${project.version};-noimport:=true + * + + + true + + + ${project.build.directory}/osgi/MANIFEST.MF + + + scp + + + iso-8859-1 + + ${commons.encoding} + + ${commons.encoding} + + ${commons.encoding} + + + https://docs.oracle.com/javase/6/docs/api/ + https://docs.oracle.com/javase/7/docs/api/ + https://docs.oracle.com/javase/8/docs/api/ + https://docs.oracle.com/javase/9/docs/api/ + https://docs.oracle.com/javase/10/docs/api/ + https://docs.oracle.com/en/java/javase/11/docs/api/ + https://docs.oracle.com/en/java/javase/12/docs/api/ + https://docs.oracle.com/en/java/javase/13/docs/api/ + https://docs.oracle.com/en/java/javase/14/docs/api/ + https://docs.oracle.com/en/java/javase/15/docs/api/ + https://docs.oracle.com/en/java/javase/16/docs/api/ + https://docs.oracle.com/en/java/javase/17/docs/api/ + https://docs.oracle.com/en/java/javase/18/docs/api/ + + ${commons.javadoc7.java.link} + + https://docs.oracle.com/javaee/5/api/ + https://docs.oracle.com/javaee/6/api/ + https://docs.oracle.com/javaee/7/api/ + + ${commons.javadoc.javaee6.link} + + + yyyy-MM-dd HH:mm:ssZ + ${scmBranch}@r${buildNumber}; ${maven.build.timestamp} + + + info + + + 100 + + + false + + + false + + 100 + + false + + + ${user.home}/commons-sites + + ${commons.componentid} + + https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-${commons.componentid} + ${commons.site.cache}/${commons.site.path} + commons.site + + + true + false + false + + + scm:svn:https://dist.apache.org/repos/dist/dev/commons/${commons.componentid} + + + ${user.name} + DEADBEEF + + https://analysis.apache.org/ + + + . + RELEASE-NOTES.txt + + + + + + + + + Commons User List + user-subscribe@commons.apache.org + user-unsubscribe@commons.apache.org + user@commons.apache.org + https://mail-archives.apache.org/mod_mbox/commons-user/ + + https://markmail.org/list/org.apache.commons.users/ + https://www.mail-archive.com/user@commons.apache.org/ + + + + Commons Dev List + dev-subscribe@commons.apache.org + dev-unsubscribe@commons.apache.org + dev@commons.apache.org + https://mail-archives.apache.org/mod_mbox/commons-dev/ + + https://markmail.org/list/org.apache.commons.dev/ + https://www.mail-archive.com/dev@commons.apache.org/ + + + + Commons Issues List + issues-subscribe@commons.apache.org + issues-unsubscribe@commons.apache.org + https://mail-archives.apache.org/mod_mbox/commons-issues/ + + https://markmail.org/list/org.apache.commons.issues/ + https://www.mail-archive.com/issues@commons.apache.org/ + + + + Commons Commits List + commits-subscribe@commons.apache.org + commits-unsubscribe@commons.apache.org + https://mail-archives.apache.org/mod_mbox/commons-commits/ + + https://markmail.org/list/org.apache.commons.commits/ + https://www.mail-archive.com/commits@commons.apache.org/ + + + + Apache Announce List + announce-subscribe@apache.org + announce-unsubscribe@apache.org + https://mail-archives.apache.org/mod_mbox/www-announce/ + + https://markmail.org/list/org.apache.announce/ + https://www.mail-archive.com/announce@apache.org/ + + + + + + + scm:git:http://gitbox.apache.org/repos/asf/commons-parent.git + scm:git:https://gitbox.apache.org/repos/asf/commons-parent.git + https://gitbox.apache.org/repos/asf?p=commons-parent.git + + + + jira + https://issues.apache.org/jira/browse/COMMONSSITE + + + + GitHub + https://github.com/apache/commons-parent/actions + + + + + + org.junit + junit-bom + ${commons.junit.version} + pom + import + + + + + + + clean apache-rat:check package site + + + + src/main/resources + + + + ${basedir} + META-INF + + NOTICE.txt + LICENSE.txt + NOTICE + LICENSE + + + + + + + + src/test/resources + + + + ${basedir} + META-INF + + NOTICE.txt + LICENSE.txt + NOTICE + LICENSE + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${commons.compiler.version} + + ${maven.compiler.source} + ${maven.compiler.target} + ${commons.encoding} + + ${commons.compiler.fork} + + ${commons.compiler.compilerVersion} + ${commons.compiler.javac} + + + + org.apache.maven.plugins + maven-assembly-plugin + ${commons.assembly-plugin.version} + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${commons.javadoc.version} + + + true + ${maven.compiler.source} + ${commons.compiler.javadoc} + ${commons.encoding} + ${commons.docEncoding} + + true + true + + ${commons.javadoc.java.link} + ${commons.javadoc.javaee.link} + + + + true + true + + + + + + + org.apache.maven.plugins + maven-remote-resources-plugin + + + true + + + + + org.apache.maven.plugins + maven-site-plugin + ${commons.site-plugin.version} + + + true + + + + + org.apache.maven.wagon + wagon-ssh + ${commons.wagon-ssh.version} + + + + + attach-descriptor + + attach-descriptor + + + + + + org.apache.maven.plugins + maven-source-plugin + ${commons.source-plugin.version} + + + + true + true + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${commons.surefire.version} + + + org.apache.maven.plugins + maven-failsafe-plugin + ${commons.failsafe.version} + + + + com.github.siom79.japicmp + japicmp-maven-plugin + ${commons.japicmp.version} + + + + ${project.groupId} + ${project.artifactId} + ${commons.bc.version} + jar + + + + + ${project.build.directory}/${project.artifactId}-${project.version}.jar + + + + true + ${commons.japicmp.breakBuildOnBinaryIncompatibleModifications} + ${commons.japicmp.breakBuildOnSourceIncompatibleModifications} + + true + true + true + ${commons.japicmp.ignoreMissingClasses} + + + METHOD_NEW_DEFAULT + true + true + PATCH + + + + + + + org.apache.commons + commons-build-plugin + ${commons.build-plugin.version} + + ${commons.release.name} + + + + org.apache.commons + commons-release-plugin + ${commons.release-plugin.version} + + + org.apache.felix + maven-bundle-plugin + ${commons.felix.version} + true + + + + biz.aQute.bnd + biz.aQute.bndlib + ${commons.biz.aQute.bndlib.version} + + + + + org.apache.rat + apache-rat-plugin + ${commons.rat.version} + + + org.codehaus.mojo + build-helper-maven-plugin + ${commons.build-helper.version} + + + org.codehaus.mojo + buildnumber-maven-plugin + ${commons.buildnumber-plugin.version} + + + org.codehaus.mojo + versions-maven-plugin + + 2.12.0 + + + org.jacoco + jacoco-maven-plugin + ${commons.jacoco.version} + + + + prepare-agent + process-test-classes + + prepare-agent + + + + report + site + + report + + + + check + + check + + + + + BUNDLE + + + CLASS + COVEREDRATIO + ${commons.jacoco.classRatio} + + + INSTRUCTION + COVEREDRATIO + ${commons.jacoco.instructionRatio} + + + METHOD + COVEREDRATIO + ${commons.jacoco.methodRatio} + + + BRANCH + COVEREDRATIO + ${commons.jacoco.branchRatio} + + + LINE + COVEREDRATIO + ${commons.jacoco.lineRatio} + + + COMPLEXITY + COVEREDRATIO + ${commons.jacoco.complexityRatio} + + + + + ${commons.jacoco.haltOnFailure} + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + ${commons.project-info.version} + + + + org.apache.bcel + bcel + 6.5.0 + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${commons.checkstyle-plugin.version} + + + com.puppycrawl.tools + checkstyle + ${commons.checkstyle.version} + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${commons.spotbugs.plugin.version} + + + com.github.spotbugs + spotbugs + ${commons.spotbugs.impl.version} + + + + + org.apache.maven.plugins + maven-pmd-plugin + ${commons.pmd.version} + + + net.sourceforge.pmd + pmd-core + ${commons.pmd-impl.version} + + + net.sourceforge.pmd + pmd-java + ${commons.pmd-impl.version} + + + net.sourceforge.pmd + pmd-javascript + ${commons.pmd-impl.version} + + + net.sourceforge.pmd + pmd-jsp + ${commons.pmd-impl.version} + + + + + org.cyclonedx + cyclonedx-maven-plugin + ${commons.cyclonedx.version} + + + package + + makeAggregateBom + + + + + library + 1.4 + true + true + true + true + true + false + false + true + all + ${project.artifactId}-${project.version}-bom + + + + org.spdx + spdx-maven-plugin + ${commons.spdx.version} + + + build-spdx + + createSPDX + + + + + + *.spdx + + + + + org.codehaus.mojo + javancss-maven-plugin + + + + + + + **/*.java + + + + + + + + + + maven-assembly-plugin + + + src/assembly/src.xml + + gnu + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + javadoc.resources + generate-sources + + run + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + true + org.apache.maven.plugins + maven-enforcer-plugin + ${commons.enforcer-plugin.version} + + + + 3.5.0 + + + ${maven.compiler.target} + + + true + + + + enforce-maven-3 + + enforce + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${commons.jar-plugin.version} + + + + test-jar + + + + true + + + + + + ${commons.manifestfile} + + ${project.name} + ${project.version} + ${project.organization.name} + ${project.name} + ${project.version} + ${project.organization.name} + org.apache + ${implementation.build} + ${maven.compiler.source} + ${maven.compiler.target} + + + + + + maven-source-plugin + + + create-source-jar + + jar-no-fork + test-jar-no-fork + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${commons.surefire.java} + + + + + org.apache.commons + commons-build-plugin + + + org.apache.felix + maven-bundle-plugin + + + + true + + ${commons.osgi.excludeDependencies} + ${project.build.directory}/osgi + + + <_nouses>true + + <_removeheaders>JAVA_1_3_HOME,JAVA_1_4_HOME,JAVA_1_5_HOME,JAVA_1_6_HOME,JAVA_1_7_HOME,JAVA_1_8_HOME,JAVA_1_9_HOME + ${commons.osgi.symbolicName} + ${commons.osgi.export} + ${commons.osgi.private} + ${commons.osgi.import} + ${commons.osgi.dynamicImport} + ${project.url} + + + + + bundle-manifest + process-classes + + manifest + + + + + + + org.apache.rat + apache-rat-plugin + + + + + site-content/** + .checkstyle + .fbprefs + .pmd + .asf.yaml + src/site/resources/download_*.cgi + src/site/resources/profile.* + profile.* + + maven-eclipse.xml + .externalToolBuilders/** + + .vscode/** + + + + + rat-check + validate + + check + + + + + + + org.apache.maven.plugins + maven-scm-publish-plugin + + ${project.reporting.outputDirectory} + scm:svn:${commons.scmPubUrl} + ${commons.scmPubCheckoutDirectory} + ${commons.scmPubServer} + true + + + + scm-publish + site-deploy + + publish-scm + + + + + + org.codehaus.mojo + versions-maven-plugin + + + org.cyclonedx + cyclonedx-maven-plugin + + + org.spdx + spdx-maven-plugin + + + + + + + + + + org.apache.maven.plugins + maven-changes-plugin + ${commons.changes.version} + + ${basedir}/src/changes/changes.xml + Fix Version,Key,Component,Summary,Type,Resolution,Status + + Fix Version DESC,Type,Key DESC + Fixed + Resolved,Closed + + Bug,New Feature,Task,Improvement,Wish,Test + + true + ${commons.changes.onlyCurrentVersion} + ${commons.changes.maxEntries} + ${commons.changes.runOnlyAtExecutionRoot} + + + + + changes-report + jira-report + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${commons.javadoc.version} + + + + default + + javadoc + + + + + + org.apache.maven.plugins + maven-jxr-plugin + ${commons.jxr.version} + + + org.apache.maven.plugins + maven-project-info-reports-plugin + ${commons.project-info.version} + + + + + index + summary + modules + + team + scm + issue-management + mailing-lists + dependency-info + dependency-management + dependencies + dependency-convergence + ci-management + + + distribution-management + + + + + + org.apache.maven.plugins + maven-site-plugin + ${commons.site-plugin.version} + + + + navigation.xml,changes.xml + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + ${commons.surefire-report.version} + + ${commons.surefire-report.aggregate} + + + + + org.apache.rat + apache-rat-plugin + ${commons.rat.version} + + + + + site-content/** + .checkstyle + .fbprefs + .pmd + .asf.yaml + src/site/resources/download_*.cgi + src/site/resources/profile.* + profile.* + + maven-eclipse.xml + .externalToolBuilders/** + + .vscode/** + + + + + + + + + svn + + + .svn + + + + + + org.codehaus.mojo + buildnumber-maven-plugin + + + validate + + create + + + + + + true + + ?????? + + + javasvn + + + + + + + + + + module-name + + + profile.module-name + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${commons.module.name} + + + + + + + + + + + parse-target-version + + + + user.home + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + parse-version + + + parse-version + + + javaTarget + ${maven.compiler.target} + + + + + + + + + + + + animal-sniffer + + + + src/site/resources/profile.noanimal + + + + + + java${javaTarget.majorVersion}${javaTarget.minorVersion} + + + + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + ${commons.animal-sniffer.version} + + + checkAPIcompatibility + + + + check + + + + + + org.codehaus.mojo.signature + ${animal-sniffer.signature} + ${commons.animal-sniffer.signature.version} + + + + + + + + + + jacoco + + + + src/site/resources/profile.jacoco + + + + + + org.jacoco + jacoco-maven-plugin + ${commons.jacoco.version} + + + + + + + org.jacoco + jacoco-maven-plugin + ${commons.jacoco.version} + + + + + report + + + + + + + + + + cobertura + + + src/site/resources/profile.cobertura + + + + + + org.codehaus.mojo + cobertura-maven-plugin + ${commons.cobertura.version} + + + + + + + japicmp + + [1.8,) + + src/site/resources/profile.japicmp + + + + + + com.github.siom79.japicmp + japicmp-maven-plugin + + + verify + + cmp + + + + + + + + + + com.github.siom79.japicmp + japicmp-maven-plugin + ${commons.japicmp.version} + + + true + ${commons.japicmp.breakBuildOnBinaryIncompatibleModifications} + ${commons.japicmp.breakBuildOnSourceIncompatibleModifications} + + true + true + true + ${commons.japicmp.ignoreMissingClasses} + + + METHOD_NEW_DEFAULT + true + true + PATCH + + + + + + + + + + + + release + + + + maven-install-plugin + + true + + + + maven-release-plugin + + + -Prelease + + + + maven-javadoc-plugin + + + create-javadoc-jar + + javadoc + jar + + package + + + + ${maven.compiler.source} + ${commons.compiler.javadoc} + + + + maven-assembly-plugin + ${commons.assembly-plugin.version} + true + + + + single + + + verify + + + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + + + org.apache.commons + commons-release-plugin + + + clean-staging + clean + + clean-staging + + + + detatch-distributions + verify + + detach-distributions + + + + stage-distributions + deploy + + stage-distributions + + + + + + + + + + + apache-release + + + + maven-release-plugin + + apache-release + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-test-sources + + test-jar + + + + + + maven-install-plugin + + true + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + + + java-1.7 + + true + 1.7 + ${JAVA_1_7_HOME}/bin/javac + ${JAVA_1_7_HOME}/bin/javadoc + ${JAVA_1_7_HOME}/bin/java + + + + + + java-1.8 + + true + 1.8 + ${JAVA_1_8_HOME}/bin/javac + ${JAVA_1_8_HOME}/bin/javadoc + ${JAVA_1_8_HOME}/bin/java + + + + + + java-1.9 + + true + 1.9 + ${JAVA_1_9_HOME}/bin/javac + ${JAVA_1_9_HOME}/bin/javadoc + ${JAVA_1_9_HOME}/bin/java + + + + + + java-1.10 + + true + 1.10 + ${JAVA_1_10_HOME}/bin/javac + ${JAVA_1_10_HOME}/bin/javadoc + ${JAVA_1_10_HOME}/bin/java + + + + + + java-1.11 + + true + 1.11 + ${JAVA_1_11_HOME}/bin/javac + ${JAVA_1_11_HOME}/bin/javadoc + ${JAVA_1_11_HOME}/bin/java + + + + + + java-1.12 + + true + 1.12 + ${JAVA_1_12_HOME}/bin/javac + ${JAVA_1_12_HOME}/bin/javadoc + ${JAVA_1_12_HOME}/bin/java + + + + + + java-1.13 + + true + 1.13 + ${JAVA_1_13_HOME}/bin/javac + ${JAVA_1_13_HOME}/bin/javadoc + ${JAVA_1_13_HOME}/bin/java + + + + + + + + test-deploy + + id::default::file:target/deploy + true + + + + + + release-notes + + + + org.apache.maven.plugins + maven-changes-plugin + ${commons.changes.version} + + + src/changes + true + ${changes.announcementDirectory} + ${changes.announcementFile} + + ${commons.release.version} + + + + + create-release-notes + generate-resources + + announcement-generate + + + + + + + + + + + svn-buildnumber + + + !buildNumber.skip + !true + + + + + + org.codehaus.mojo + buildnumber-maven-plugin + + + generate-resources + + create + + + + + + true + + ?????? + false + false + + + + + + + + javasvn + + + + org.codehaus.mojo + buildnumber-maven-plugin + + + javasvn + + + + + + + + + jdk7-plugin-fix-version + + [1.7,1.8) + + + 3.5.1 + 1.17 + 3.5.0 + + + + + + site-basic + + true + true + true + true + true + true + true + true + + + + + travis-cobertura + + + + org.codehaus.mojo + cobertura-maven-plugin + ${commons.cobertura.version} + + + xml + + + + + org.eluder.coveralls + coveralls-maven-plugin + ${commons.coveralls.version} + + ${commons.coveralls.timestampFormat} + + + + + + + + travis-jacoco + + + + org.jacoco + jacoco-maven-plugin + ${commons.jacoco.version} + + + org.eluder.coveralls + coveralls-maven-plugin + ${commons.coveralls.version} + + ${commons.coveralls.timestampFormat} + + + + + + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom new file mode 100644 index 00000000000..de9e3fc418d --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom @@ -0,0 +1,159 @@ + + + + + + + + 4.0.0 + org.junit + junit-bom + 5.9.0 + pom + JUnit 5 (Bill of Materials) + This Bill of Materials POM can be used to ease dependency management when referencing multiple JUnit artifacts using Gradle or Maven. + https://junit.org/junit5/ + + + Eclipse Public License v2.0 + https://www.eclipse.org/legal/epl-v20.html + + + + + bechte + Stefan Bechtold + stefan.bechtold@me.com + + + jlink + Johannes Link + business@johanneslink.net + + + marcphilipp + Marc Philipp + mail@marcphilipp.de + + + mmerdes + Matthias Merdes + matthias.merdes@heidelpay.com + + + sbrannen + Sam Brannen + sam@sambrannen.com + + + sormuras + Christian Stein + sormuras@gmail.com + + + juliette-derancourt + Juliette de Rancourt + derancourt.juliette@gmail.com + + + + scm:git:git://github.com/junit-team/junit5.git + scm:git:git://github.com/junit-team/junit5.git + https://github.com/junit-team/junit5 + + + + + org.junit.jupiter + junit-jupiter + 5.9.0 + + + org.junit.jupiter + junit-jupiter-api + 5.9.0 + + + org.junit.jupiter + junit-jupiter-engine + 5.9.0 + + + org.junit.jupiter + junit-jupiter-migrationsupport + 5.9.0 + + + org.junit.jupiter + junit-jupiter-params + 5.9.0 + + + org.junit.platform + junit-platform-commons + 1.9.0 + + + org.junit.platform + junit-platform-console + 1.9.0 + + + org.junit.platform + junit-platform-engine + 1.9.0 + + + org.junit.platform + junit-platform-jfr + 1.9.0 + + + org.junit.platform + junit-platform-launcher + 1.9.0 + + + org.junit.platform + junit-platform-reporting + 1.9.0 + + + org.junit.platform + junit-platform-runner + 1.9.0 + + + org.junit.platform + junit-platform-suite + 1.9.0 + + + org.junit.platform + junit-platform-suite-api + 1.9.0 + + + org.junit.platform + junit-platform-suite-commons + 1.9.0 + + + org.junit.platform + junit-platform-suite-engine + 1.9.0 + + + org.junit.platform + junit-platform-testkit + 1.9.0 + + + org.junit.vintage + junit-vintage-engine + 5.9.0 + + + + From 7f1c79d55c98dda6a3a4d43a29af75fc526e0e05 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Mon, 8 Apr 2024 19:47:15 +0200 Subject: [PATCH 14/42] Fix unit tests by resetting caches before test Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/parse_pom_xml_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 61f001cb631..314c6c36af8 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -20,6 +20,7 @@ import ( ) func Test_parserPomXML(t *testing.T) { + resetRepoUtils() tests := []struct { input string expected []pkg.Package @@ -161,6 +162,7 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { } func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { + resetRepoUtils() // Using the local repository, the version of junit-jupiter will be resolved expectedPackages := getCommonsTextExpectedPackages() @@ -177,6 +179,10 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { }, } } + // if i < 3 { + // &expectedPackages[i].Licenses + + // } } tests := []struct { @@ -209,6 +215,7 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { } func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { + resetRepoUtils() mux, url, teardown := setup() defer teardown() // Using the local repository, the version of junit-jupiter will be resolved @@ -276,6 +283,7 @@ func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { } func Test_parsePomXMLProject(t *testing.T) { + resetRepoUtils() // TODO: ideally we would have the path to the contained pom.xml, not the jar jarLocation := file.NewLocation("path/to/archive.jar") tests := []struct { @@ -675,3 +683,10 @@ func getCommonsTextExpectedPackages() []pkg.Package { }, } } + +// Reset caches in maven_repo_utils for independent unit tests. +func resetRepoUtils() { + parsedPomFilesCache = make(map[mavenCoordinate]*gopom.Project) + checkedForMavenLocalRepo = false + mavenLocalRepoDir = "" +} From c72776535b16efdc9522922f3d4d151e77d8d218 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:21:55 +0200 Subject: [PATCH 15/42] Recurse into parent poms with cycle detection Update configuration documentation Improve maven groupid detection Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- README.md | 20 +++-- cmd/syft/internal/options/java.go | 2 - syft/pkg/cataloger/java/archive_parser.go | 71 ++++++++++-------- .../pkg/cataloger/java/archive_parser_test.go | 3 +- syft/pkg/cataloger/java/config.go | 9 +-- syft/pkg/cataloger/java/maven_repo_utils.go | 74 ++++++++++++------- syft/pkg/cataloger/java/parse_pom_xml.go | 2 +- syft/pkg/cataloger/java/parse_pom_xml_test.go | 4 - 8 files changed, 107 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 96ad78ac916..3e453a5e02c 100644 --- a/README.md +++ b/README.md @@ -711,15 +711,25 @@ golang: from-contents: true java: + # the remote repository to use for downloading pom files. If you have a caching proxy, such as + # Artifactory or Nexus Repository server available, you are advised to use that server for + # improved speed and limiting the amount of requests to Maven Central. + # e.g.: https://nexus.example.net/repository/maven-public/ maven-url: "https://repo1.maven.org/maven2" - max-parent-recursive-depth: 5 - # enables Syft to use the network to fill in more detailed information about artifacts - # currently this enables searching maven-url for license data - # when running across pom.xml files that could have more information, syft will - # explicitly search maven for license information by querying the online pom when this is true + # enables Syft to use the network to fill in more detailed information about artifacts. + # this enables searching maven-url for dependency and license data + # when running across pom.xml files that could have more information, syft will explicitly search + # maven for dependency and license information by querying the online pom when this is true. # this option is helpful for when the parent pom has more data, # that is not accessible from within the final built artifact use-network: false + # use the local Maven repository to retrieve pom files. When Maven is installed and was used + # for building the software that is being scanned, then most pom files will be available in this + # repository on the local file system. + use-maven-localrepository: true + # override the default location of the local Maven repository. + # the default is the subdirectory '.m2/repository' in your home dir. + MavenLocalRepositoryDir: /.m2/repository linux-kernel: # whether to catalog linux kernel modules found within lib/modules/** directories diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index bb9697f055d..e98949e7eb4 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -7,7 +7,6 @@ type javaConfig struct { UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"` - MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } func defaultJavaConfig() javaConfig { @@ -18,6 +17,5 @@ func defaultJavaConfig() javaConfig { UseMavenLocalRepository: def.UseMavenLocalRepository, MavenLocalRepositoryDir: def.MavenLocalRepositoryDir, MavenURL: def.MavenBaseURL, - MaxParentRecursiveDepth: def.MaxParentRecursiveDepth, } } diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 3f98068d265..01a8ff7c3ee 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -221,7 +221,7 @@ func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaMan 3. manifest 4. filename */ - name, version, pomLicenses := j.guessMainPackageNameAndVersionFromPomInfo(ctx, j.cfg) + group, name, version, pomLicenses := j.guessMainPackageNameAndVersionFromPomInfo(ctx, j.cfg) if name == "" { name = selectName(manifest, j.fileInfo) } @@ -247,24 +247,29 @@ func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaMan // If we didn't find any licenses in the archive so far, we'll try again in Maven Central using groupIDFromJavaMetadata if len(licenses) == 0 && j.cfg.UseNetwork { - licenses = findLicenseFromJavaMetadata(ctx, name, manifest, version, j, licenses) + licenses = findLicenseFromJavaMetadata(ctx, group, name, manifest, version, j, licenses) } return licenses, name, version, nil } -func findLicenseFromJavaMetadata(ctx context.Context, name string, manifest *pkg.JavaManifest, version string, j *archiveParser, licenses []pkg.License) []pkg.License { - var groupID = name - if gID := groupIDFromJavaMetadata(name, pkg.JavaArchive{Manifest: manifest}); gID != "" { - groupID = gID +func findLicenseFromJavaMetadata(ctx context.Context, group, name string, manifest *pkg.JavaManifest, version string, j *archiveParser, licenses []pkg.License) []pkg.License { + var groupID = group + if group == "" { + if gID := groupIDFromJavaMetadata(name, pkg.JavaArchive{Manifest: manifest}); gID != "" { + groupID = gID + } } - pomLicenses := recursivelyFindLicensesFromParentPom(ctx, groupID, name, version, j.cfg) - if len(pomLicenses) == 0 { + pomLicenses, foundPom := recursivelyFindLicensesFromParentPom(ctx, groupID, name, version, j.cfg) + log.Tracef("[REMOVE] 1 findLicenseFromJavaMetadata, coordinates %s:%s:%s", groupID, name, version) + + if !foundPom && len(pomLicenses) == 0 { // Try removing the last part of the groupId, as sometimes it duplicates the artifactId packages := strings.Split(groupID, ".") groupID = strings.Join(packages[:len(packages)-1], ".") - pomLicenses = recursivelyFindLicensesFromParentPom(ctx, groupID, name, version, j.cfg) + log.Tracef("[REMOVE] 2 findLicenseFromJavaMetadata, coordinates %s:%s:%s", groupID, name, version) + pomLicenses, _ = recursivelyFindLicensesFromParentPom(ctx, groupID, name, version, j.cfg) } if len(pomLicenses) > 0 { @@ -281,7 +286,8 @@ type parsedPomProject struct { Licenses []pkg.License } -func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Context, cfg ArchiveCatalogerConfig) (name, version string, licenses []pkg.License) { +// Try to find artifact group, id, version and license(s) +func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Context, cfg ArchiveCatalogerConfig) (group, name, version string, licenses []pkg.License) { pomPropertyMatches := j.fileManifest.GlobMatch(false, pomPropertiesGlob) pomMatches := j.fileManifest.GlobMatch(false, pomXMLGlob) var pomPropertiesObject pkg.JavaPomProperties @@ -301,6 +307,10 @@ func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Co } } + group = pomPropertiesObject.GroupID + if group == "" && pomProjectObject != nil { + group = pomProjectObject.GroupID + } name = pomPropertiesObject.ArtifactID if name == "" && pomProjectObject != nil { name = pomProjectObject.ArtifactID @@ -309,25 +319,24 @@ func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Co if version == "" && pomProjectObject != nil { version = pomProjectObject.Version } - if j.cfg.UseNetwork { - if pomProjectObject == nil { - // If we have no pom.xml, check maven central using pom.properties - parentLicenses := recursivelyFindLicensesFromParentPom(ctx, pomPropertiesObject.GroupID, pomPropertiesObject.ArtifactID, pomPropertiesObject.Version, j.cfg) - if len(parentLicenses) > 0 { - for _, licenseName := range parentLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) - } + + if pomProjectObject == nil { + // If we have no pom.xml, check maven central using pom.properties + parentLicenses, _ := recursivelyFindLicensesFromParentPom(ctx, pomPropertiesObject.GroupID, pomPropertiesObject.ArtifactID, pomPropertiesObject.Version, j.cfg) + if len(parentLicenses) > 0 { + for _, licenseName := range parentLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } - } else { - findPomLicenses(ctx, pomProjectObject, j.cfg) } + } else { + findPomLicenses(ctx, pomProjectObject, j.cfg) } if pomProjectObject != nil { licenses = pomProjectObject.Licenses } - return name, version, licenses + return group, name, version, licenses } func artifactIDMatchesFilename(artifactID, fileName string) bool { @@ -340,7 +349,7 @@ func artifactIDMatchesFilename(artifactID, fileName string) bool { func findPomLicenses(ctx context.Context, pomProjectObject *parsedPomProject, cfg ArchiveCatalogerConfig) { // If we don't have any licenses until now, and if we have a parent Pom, then we'll check the parent pom in maven central for licenses. if pomProjectObject != nil && pomProjectObject.Parent != nil && len(pomProjectObject.Licenses) == 0 { - parentLicenses := recursivelyFindLicensesFromParentPom( + parentLicenses, _ := recursivelyFindLicensesFromParentPom( ctx, pomProjectObject.Parent.GroupID, pomProjectObject.Parent.ArtifactID, @@ -600,18 +609,16 @@ func newPackageFromMavenData(ctx context.Context, pomProperties pkg.JavaPomPrope var pkgPomProject *pkg.JavaPomProject licenses := make([]pkg.License, 0) - if cfg.UseNetwork { - if parsedPomProject == nil { - // If we have no pom.xml, check maven central using pom.properties - parentLicenses := recursivelyFindLicensesFromParentPom(ctx, pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version, cfg) - if len(parentLicenses) > 0 { - for _, licenseName := range parentLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) - } + if parsedPomProject == nil { + // If we have no pom.xml, check maven central using pom.properties + parentLicenses, _ := recursivelyFindLicensesFromParentPom(ctx, pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version, cfg) + if len(parentLicenses) > 0 { + for _, licenseName := range parentLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } - } else { - findPomLicenses(ctx, parsedPomProject, cfg) } + } else { + findPomLicenses(ctx, parsedPomProject, cfg) } if parsedPomProject != nil { diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index d435e17bb05..f63d3bace96 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -93,7 +93,6 @@ func TestSearchMavenForLicenses(t *testing.T) { UseNetwork: true, UseMavenLocalRepository: false, MavenBaseURL: url, - MaxParentRecursiveDepth: 2, }, requestHandlers: []handlerPath{ { @@ -139,7 +138,7 @@ func TestSearchMavenForLicenses(t *testing.T) { defer cleanupFn() // assert licenses are discovered from upstream - _, _, licenses := ap.guessMainPackageNameAndVersionFromPomInfo(context.Background(), tc.config) + _, _, _, licenses := ap.guessMainPackageNameAndVersionFromPomInfo(context.Background(), tc.config) assert.Equal(t, tc.expectedLicenses, licenses) }) } diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 78bf78c1dde..4a75358497f 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -10,18 +10,16 @@ type ArchiveCatalogerConfig struct { UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepositoryk"` MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` MavenBaseURL string `yaml:"maven-base-url" json:"maven-base-url" mapstructure:"maven-base-url"` - MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { - localRepoDir, _ := GetDefaultMavenLocalRepoLocation() + localRepoDir, _ := getDefaultMavenLocalRepoLocation() return ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.DefaultArchiveSearchConfig(), UseNetwork: false, UseMavenLocalRepository: true, MavenLocalRepositoryDir: localRepoDir, MavenBaseURL: mavenBaseURL, - MaxParentRecursiveDepth: 5, } } @@ -47,10 +45,7 @@ func (j ArchiveCatalogerConfig) WithMavenBaseURL(input string) ArchiveCatalogerC return j } -func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSearchConfig, maxDepth int) ArchiveCatalogerConfig { - if maxDepth > 0 { - j.MaxParentRecursiveDepth = maxDepth - } +func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSearchConfig) ArchiveCatalogerConfig { j.ArchiveSearchConfig = search return j } diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index c88e17fcec2..a187cebf8cf 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -46,32 +46,32 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque // Try to find the version of a dependency (groupID, artifactID) by parsing all parent poms and imported managed dependencies (maven BOMs). // Properties are gathered in the order that they are encountered: in Maven the latest definition of a property (highest in hierarchy) is used. -// parsedPomFiles contains all previously parsed pom files encountered by earlier invocations of this function on the stack. So for the first -// call parsedPomFiles should be nil. It is used to prevent cycles (endless loops). +// processedPomFiles contains all previously processed pom files encountered by earlier invocations of this function on the stack. So for the first +// call processedPomFiles should be nil. It is used to prevent cycles (endless loops). func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[mavenCoordinate]bool) string { + pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, processedPomFiles map[mavenCoordinate]bool) string { - // Create map to keep track of parsed pom files and to prevent cycles. - if parsedPomFiles == nil { - parsedPomFiles = make(map[mavenCoordinate]bool) + // Create map to keep track of processed pom files and to prevent cycles. + if processedPomFiles == nil { + processedPomFiles = make(map[mavenCoordinate]bool) } log.Debugf("recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %d", - findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version, len(parsedPomFiles)) + findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version, len(processedPomFiles)) pomCoordinates := mavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} - _, alreadyParsed := parsedPomFiles[pomCoordinates] - if alreadyParsed { + _, alreadyProcessed := processedPomFiles[pomCoordinates] + if alreadyProcessed { log.Debug("skipping already processed pom.") return "" } else { - parsedPomFiles[pomCoordinates] = true + processedPomFiles[pomCoordinates] = true } addMissingPropertiesFromProject(allProperties, pom) foundVersion := "" if pom.DependencyManagement != nil { foundVersion = findVersionInDependencyManagement( - ctx, findGroupID, findArtifactID, pom, cfg, allProperties, parsedPomFiles) + ctx, findGroupID, findArtifactID, pom, cfg, allProperties, processedPomFiles) } if isPropertyResolved(foundVersion) { return foundVersion @@ -90,7 +90,7 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup addMissingPropertiesFromProject(allProperties, parentPom) addPropertiesToProject(parentPom, allProperties) foundVersion = recursivelyFindVersionFromManagedOrInherited( - ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, parsedPomFiles) + ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, processedPomFiles) } else { log.Warnf("unable to get parent pom [%s, %s, %s]: %v", parentGroupID, parentArtifactID, parentVersion, err) @@ -113,7 +113,7 @@ func isPropertyResolved(value string) bool { // Find given dependency (groupID, artifactID) in the dependencyManagement section of project 'pom'. // May recursively call recursivelyFindVersionFromManagedOrInherited when a Maven BOM is found. func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, parsedPomFiles map[mavenCoordinate]bool) string { + pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, processedPomFiles map[mavenCoordinate]bool) string { for _, dependency := range *getPomManagedDependencies(pom) { log.Tracef("got managed dependency: [%s, %s, %s]", @@ -130,7 +130,7 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt if err == nil && bomProject != nil { foundVersion := recursivelyFindVersionFromManagedOrInherited( - ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, parsedPomFiles) + ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, processedPomFiles) log.Tracef("finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) @@ -160,30 +160,48 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt return "" } -func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { - log.Debugf("recursively finding license from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", +// Search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. +func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) ([]string, bool) { + log.Debugf("recursively finding licenses from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", groupID, artifactID, groupID, artifactID, version) var licenses []string - // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth - for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { + var foundPom bool = false + processedPomFiles := make(map[mavenCoordinate]bool) + + // As there can be nested parent poms, we'll recursively check for licenses until no parent is found + recursionLevel := 0 + for safeString(&artifactID) != "" { + log.Tracef("recursively find licenses for [%s, %s, %s], recursion level: %d", groupID, artifactID, version, recursionLevel) + recursionLevel++ + parentPom, err := getPomFromCacheOrMaven(ctx, groupID, artifactID, version, make(map[string]string), cfg) if err != nil { // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - log.Tracef("unable to get parent pom from Maven repository: %v", err) - return []string{} + log.Tracef("unable to get parent pom: %v", err) + return []string{}, foundPom } parentLicenses := parseLicensesFromPom(parentPom) if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { licenses = parentLicenses + foundPom = true break } + // Check for cycle and store processed pom ID when not + pomCoordinates := mavenCoordinate{*parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version} + _, alreadyProcessed := processedPomFiles[pomCoordinates] + if alreadyProcessed { + log.Debug("already processed parent pom, stop searching.") + break + } else { + processedPomFiles[pomCoordinates] = true + } - groupID = *parentPom.Parent.GroupID - artifactID = *parentPom.Parent.ArtifactID - version = *parentPom.Parent.Version + groupID = pomCoordinates.GroupID + artifactID = pomCoordinates.ArtifactID + version = pomCoordinates.Version } - return licenses + return licenses, foundPom } // Get a parent pom from cache, local repository or download from a Maven repository @@ -286,7 +304,7 @@ func getLocalRepositoryExists(cfg ArchiveCatalogerConfig) (string, bool) { } // Get default location of the Maven local repository at /.m2/repository -func GetDefaultMavenLocalRepoLocation() (string, error) { +func getDefaultMavenLocalRepoLocation() (string, error) { homeDir, err := os.UserHomeDir() if err != nil { @@ -305,6 +323,12 @@ func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mave if len(groupID) == 0 || len(artifactID) == 0 || !isPropertyResolved(version) { return nil, fmt.Errorf("missing/incomplete maven artifact coordinates: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) } + // Downloading snapshots requires additional steps to determine the latest snapshot version. + // See: https://maven.apache.org/ref/3-LATEST/maven-repository-metadata/ + if strings.HasSuffix(version, "-SNAPSHOT") { + return nil, fmt.Errorf("downloading snapshot artifacts is not supported: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) + } + requestURL, err := formatMavenPomURL(groupID, artifactID, version, mavenBaseURL) if err != nil { return nil, err diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 7e35411b55e..e743bfb9491 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -186,7 +186,7 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende } if isPropertyResolved(version) { - parentLicenses := recursivelyFindLicensesFromParentPom( + parentLicenses, _ := recursivelyFindLicensesFromParentPom( ctx, m.PomProperties.GroupID, m.PomProperties.ArtifactID, diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 314c6c36af8..2816e50ba01 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -179,10 +179,6 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { }, } } - // if i < 3 { - // &expectedPackages[i].Licenses - - // } } tests := []struct { From 26587c837a3ce4e3bb5597c56e08377c443bac95 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:04:03 +0200 Subject: [PATCH 16/42] fix bug: missed changed function signature Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- cmd/syft/internal/options/catalog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index b20b9ee5115..1fc559faba0 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -190,7 +190,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository). WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir). WithMavenBaseURL(cfg.Java.MavenURL). - WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth), + WithArchiveTraversal(archiveSearch), } } From bf8ebfdb9d6cb55ec5aaa83ff4bee2bc2e0c4a07 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:39:54 +0200 Subject: [PATCH 17/42] Remove unneeded logging Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- cmd/syft/internal/options/catalog.go | 38 +----------------------- syft/pkg/cataloger/java/parse_pom_xml.go | 32 -------------------- 2 files changed, 1 insertion(+), 69 deletions(-) diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 1fc559faba0..da8d4c9cbba 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -5,10 +5,7 @@ import ( "sort" "strings" - "github.com/gookit/color" "github.com/iancoleman/strcase" - "github.com/pborman/indent" - "gopkg.in/yaml.v3" "github.com/anchore/clio" "github.com/anchore/fangs" @@ -75,8 +72,7 @@ func DefaultCatalog() Catalog { } func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { - - config := syft.DefaultCreateSBOMConfig(). + return syft.DefaultCreateSBOMConfig(). WithTool(id.Name, id.Version). WithParallelism(cfg.Parallelism). WithRelationshipsConfig(cfg.ToRelationshipsConfig()). @@ -88,38 +84,6 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithDefaults(cfg.DefaultCatalogers...). WithExpression(cfg.SelectCatalogers...), ) - - // log.Tracef("scan SBOMConfig: %+v", config) - logConfiguration(config.Packages.JavaArchive) - return config -} - -func logConfiguration(cfg java.ArchiveCatalogerConfig) { - var sb strings.Builder - - var str string - // yaml is pretty human friendly (at least when compared to json) - cfgBytes, err := yaml.Marshal(cfg) - if err != nil { - str = fmt.Sprintf("%+v", err) - } else { - str = string(cfgBytes) - } - - str = strings.TrimSpace(str) - - if str != "" && str != "{}" { - sb.WriteString(str + "\n") - } - - content := sb.String() - - if content != "" { - formatted := color.Magenta.Sprint(indent.String(" ", strings.TrimSpace(content))) - log.Debugf("config:\n%+v", formatted) - } else { - log.Debug("config: (none)") - } } func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index e743bfb9491..fb5cf9195dd 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -10,12 +10,9 @@ import ( "regexp" "strings" - "github.com/gookit/color" - "github.com/pborman/indent" "github.com/saintfish/chardet" "github.com/vifraa/gopom" "golang.org/x/net/html/charset" - "gopkg.in/yaml.v2" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" @@ -28,36 +25,7 @@ const pomXMLGlob = "*pom.xml" var propertyMatcher = regexp.MustCompile("[$][{][^}]+[}]") -func logConfiguration(cfg ArchiveCatalogerConfig) { - var sb strings.Builder - - var str string - // yaml is pretty human friendly (at least when compared to json) - cfgBytes, err := yaml.Marshal(cfg) - if err != nil { - str = fmt.Sprintf("%+v", err) - } else { - str = string(cfgBytes) - } - - str = strings.TrimSpace(str) - - if str != "" && str != "{}" { - sb.WriteString(str + "\n") - } - - content := sb.String() - - if content != "" { - formatted := color.Magenta.Sprint(indent.String(" ", strings.TrimSpace(content))) - log.Debugf("config:\n%+v", formatted) - } else { - log.Debug("config: (none)") - } -} - func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - logConfiguration(gap.cfg) pom, err := decodePomXML(reader) if err != nil { return nil, nil, err From eb80105abef9cb8ec578992e4a6dd1c9b27824f5 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Wed, 10 Apr 2024 12:14:39 +0200 Subject: [PATCH 18/42] remove unused/duplicate modules Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index f7c0967e97e..b58bc9c95d8 100644 --- a/go.mod +++ b/go.mod @@ -232,7 +232,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.41.0 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/go.sum b/go.sum index 107c018f9ab..11c5782dfcd 100644 --- a/go.sum +++ b/go.sum @@ -1335,7 +1335,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From ec208f9fa3e0d0d34d87a0d5fdc621c2c02ae7c1 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Wed, 10 Apr 2024 12:15:50 +0200 Subject: [PATCH 19/42] Retry resolving version property after processing all parent poms Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 30 ++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index a187cebf8cf..c502fb52450 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -68,16 +68,17 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup } addMissingPropertiesFromProject(allProperties, pom) - foundVersion := "" + foundDepMngVersion := "" if pom.DependencyManagement != nil { - foundVersion = findVersionInDependencyManagement( + foundDepMngVersion = findVersionInDependencyManagement( ctx, findGroupID, findArtifactID, pom, cfg, allProperties, processedPomFiles) } - if isPropertyResolved(foundVersion) { - return foundVersion + if isPropertyResolved(foundDepMngVersion) { + return foundDepMngVersion } // If a parent exists, search it recursively. + foundRecVersion := "" if pom.Parent != nil { parentGroupID := *pom.Parent.GroupID parentArtifactID := *pom.Parent.ArtifactID @@ -87,16 +88,27 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup if parentPom != nil { log.Debugf("found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) - addMissingPropertiesFromProject(allProperties, parentPom) - addPropertiesToProject(parentPom, allProperties) - foundVersion = recursivelyFindVersionFromManagedOrInherited( + foundRecVersion = recursivelyFindVersionFromManagedOrInherited( ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, processedPomFiles) + addMissingPropertiesFromProject(allProperties, pom) + addPropertiesToProject(pom, allProperties) } else { log.Warnf("unable to get parent pom [%s, %s, %s]: %v", parentGroupID, parentArtifactID, parentVersion, err) } } + foundVersion := resolveProperty(*pom, &foundRecVersion, getPropertyName(foundRecVersion)) + + // foundDepMngVersion may contain the version in a property that could not previously be resolved. + if !isPropertyResolved(foundVersion) && foundDepMngVersion != "" { + foundDepMngVersion = resolveProperty(*pom, &foundDepMngVersion, getPropertyName(foundDepMngVersion)) + + if isPropertyResolved(foundDepMngVersion) { + foundVersion = foundDepMngVersion + } + } + if foundVersion == "" { log.Tracef("no version found for dependency: [%s, %s]", findGroupID, findArtifactID) } else { @@ -154,6 +166,10 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt log.Debugf("found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) return foundVersion } + if strings.HasPrefix(foundVersion, "${") { + log.Tracef("found version in property reference for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) + return foundVersion + } } } log.Tracef("dependency not found in dependencyManagement") From d3efe1fa5eae4d4f67d1ca3f4a4bcc511b9407c5 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:00:44 +0200 Subject: [PATCH 20/42] Update instructions for java configuration Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9fa1b67a86f..a7688a06a5c 100644 --- a/README.md +++ b/README.md @@ -713,18 +713,18 @@ java: # the remote repository to use for downloading pom files. If you have a caching proxy, such as # Artifactory or Nexus Repository server available, you are advised to use that server for # improved speed and limiting the amount of requests to Maven Central. - # e.g.: https://nexus.example.net/repository/maven-public/ + # e.g.: https://nexus.mycompany.net/repository/maven-public/ maven-url: "https://repo1.maven.org/maven2" - # enables Syft to use the network to fill in more detailed information about artifacts. - # this enables searching maven-url for dependency and license data - # when running across pom.xml files that could have more information, syft will explicitly search - # maven for dependency and license information by querying the online pom when this is true. - # this option is helpful for when the parent pom has more data, - # that is not accessible from within the final built artifact + # enables Syft to use the network to fetch version and license information for packages when + # a parent or imported pom file is not found in the local maven repository. + # the pom files are downloaded from the remote Maven repository at `maven-url`. use-network: false - # use the local Maven repository to retrieve pom files. When Maven is installed and was used + # use the local Maven repository to retrieve pom files. When Maven is installed and was previously used # for building the software that is being scanned, then most pom files will be available in this - # repository on the local file system. + # repository on the local file system. this greatly speeds up scans. when all pom files are available + # in the local repository, then `use-network` is not needed. + # TIP: If you want to download all required pom files to the local repository without running a full + # build, run `mvn help:effective-pom` before performing the scan with syft. use-maven-localrepository: true # override the default location of the local Maven repository. # the default is the subdirectory '.m2/repository' in your home dir. From 1df85832784990ebf10ea769f3ab482e062838a7 Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:50:05 +0200 Subject: [PATCH 21/42] remove accidentally created SBOM files Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- cmd/syft/sbom.cyclonedx.build.json | 0 sbom.cyclonedx.build.json | 1 - 2 files changed, 1 deletion(-) delete mode 100644 cmd/syft/sbom.cyclonedx.build.json delete mode 100644 sbom.cyclonedx.build.json diff --git a/cmd/syft/sbom.cyclonedx.build.json b/cmd/syft/sbom.cyclonedx.build.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sbom.cyclonedx.build.json b/sbom.cyclonedx.build.json deleted file mode 100644 index e8b76edb910..00000000000 --- a/sbom.cyclonedx.build.json +++ /dev/null @@ -1 +0,0 @@ -{"$schema":"http://cyclonedx.org/schema/bom-1.5.schema.json","bomFormat":"CycloneDX","specVersion":"1.5","serialNumber":"urn:uuid:3c306081-82af-4400-9997-63e7078d7b7c","version":1,"metadata":{"timestamp":"2024-04-04T14:17:14+02:00","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"[not provided]"}]},"component":{"bom-ref":"2e2aaf91e5ee9005","type":"file","name":"/home/gcalis/data/httpcomponents-core"}},"components":[{"bom-ref":"pkg:github/actions/cache@v3?package-id=ea5fbdd0bd953c00","type":"library","name":"actions/cache","version":"v3","cpe":"cpe:2.3:a:actions\\/cache:actions\\/cache:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/cache@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/maven.yml"}]},{"bom-ref":"pkg:github/actions/checkout@v3?package-id=9bf49ea83d8e4721","type":"library","name":"actions/checkout","version":"v3","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/codeql-analysis.yml"}]},{"bom-ref":"pkg:github/actions/checkout@v3?package-id=17715560247b3075","type":"library","name":"actions/checkout","version":"v3","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/depsreview.yaml"}]},{"bom-ref":"pkg:github/actions/checkout@v3?package-id=0de54a2b1c1ae315","type":"library","name":"actions/checkout","version":"v3","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:location:0:path","value":"/.github/workflows/maven.yml"}]},{"bom-ref":"pkg:github/actions/dependency-review-action@v3?package-id=d102d9940748538d","type":"library","name":"actions/dependency-review-action","version":"v3","cpe":"cpe:2.3:a:actions\\/dependency-review-action:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/dependency-review-action@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency-review-action:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review_action:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review_action:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency-review:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency-review:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency_review:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency:actions\\/dependency-review-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/dependency:actions\\/dependency_review_action:v3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/depsreview.yaml"}]},{"bom-ref":"pkg:github/actions/setup-java@v3?package-id=9c23239543436d31","type":"library","name":"actions/setup-java","version":"v3","cpe":"cpe:2.3:a:actions\\/setup-java:actions\\/setup-java:v3:*:*:*:*:*:*:*","purl":"pkg:github/actions/setup-java@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup-java:actions\\/setup_java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup_java:actions\\/setup-java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup_java:actions\\/setup_java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup:actions\\/setup-java:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup:actions\\/setup_java:v3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/maven.yml"}]},{"bom-ref":"pkg:maven/commons-cli/commons-cli@1.5.0?package-id=010b3a6cfd53fe6d","type":"library","group":"commons-cli","name":"commons-cli","version":"1.5.0","cpe":"cpe:2.3:a:commons-cli:commons-cli:1.5.0:*:*:*:*:*:*:*","purl":"pkg:maven/commons-cli/commons-cli@1.5.0","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons-cli:commons_cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons_cli:commons-cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons_cli:commons_cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons:commons-cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:commons:commons_cli:1.5.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"commons-cli"},{"name":"syft:metadata:-:groupID","value":"commons-cli"}]},{"bom-ref":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber?package-id=c0f757c783b64a19","type":"library","group":"org.conscrypt","name":"conscrypt-openjdk-uber","cpe":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"conscrypt-openjdk-uber"},{"name":"syft:metadata:-:groupID","value":"org.conscrypt"}]},{"bom-ref":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber?package-id=71901d19daed7d37","type":"library","group":"org.conscrypt","name":"conscrypt-openjdk-uber","cpe":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.conscrypt/conscrypt-openjdk-uber","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk-uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk_uber:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt-openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt_openjdk:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt-openjdk-uber:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:conscrypt:conscrypt_openjdk_uber:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"conscrypt-openjdk-uber"},{"name":"syft:metadata:-:groupID","value":"org.conscrypt"}]},{"bom-ref":"pkg:github/github/codeql-action@v2?package-id=9579f159130ae8bd#analyze","type":"library","name":"github/codeql-action/analyze","version":"v2","cpe":"cpe:2.3:a:github\\/codeql-action\\/analyze:github\\/codeql-action\\/analyze:v2:*:*:*:*:*:*:*","purl":"pkg:github/github/codeql-action@v2#analyze","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql-action\\/analyze:github\\/codeql_action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/analyze:github\\/codeql-action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/analyze:github\\/codeql_action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql-action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql_action\\/analyze:v2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/codeql-analysis.yml"}]},{"bom-ref":"pkg:github/github/codeql-action@v2?package-id=9dd3e55e255449f9#init","type":"library","name":"github/codeql-action/init","version":"v2","cpe":"cpe:2.3:a:github\\/codeql-action\\/init:github\\/codeql-action\\/init:v2:*:*:*:*:*:*:*","purl":"pkg:github/github/codeql-action@v2#init","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql-action\\/init:github\\/codeql_action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/init:github\\/codeql-action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql_action\\/init:github\\/codeql_action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql-action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:github\\/codeql:github\\/codeql_action\\/init:v2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/codeql-analysis.yml"}]},{"bom-ref":"pkg:maven/org.hamcrest/hamcrest?package-id=8eb194cb2ac4021d","type":"library","group":"org.hamcrest","name":"hamcrest","cpe":"cpe:2.3:a:hamcrest:hamcrest:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.hamcrest/hamcrest","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"hamcrest"},{"name":"syft:metadata:-:groupID","value":"org.hamcrest"}]},{"bom-ref":"pkg:maven/org.hamcrest/hamcrest?package-id=9dcf00765a01f8f0","type":"library","group":"org.hamcrest","name":"hamcrest","cpe":"cpe:2.3:a:hamcrest:hamcrest:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.hamcrest/hamcrest","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"hamcrest"},{"name":"syft:metadata:-:groupID","value":"org.hamcrest"}]},{"bom-ref":"pkg:maven/org.hamcrest/hamcrest?package-id=a6bc2f001edcdf55","type":"library","group":"org.hamcrest","name":"hamcrest","cpe":"cpe:2.3:a:hamcrest:hamcrest:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.hamcrest/hamcrest","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"hamcrest"},{"name":"syft:metadata:-:groupID","value":"org.hamcrest"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5?package-id=06f452ca662df3d1","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","cpe":"cpe:2.3:a:apache:httpcore5:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5?package-id=09c4a0e9860a560a","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","cpe":"cpe:2.3:a:apache:httpcore5:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5?package-id=ac453cdfa50cbd06","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","cpe":"cpe:2.3:a:apache:httpcore5:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4?package-id=61d83365dff554f1","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"1a909879fee9bad815223190967d9199de79297a"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/target/httpcore5-5.2.4-tests.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5/target/httpcore5-5.2.4-tests.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4?package-id=2b3b8d3364a0dbbb","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"3af020c5a3295abecfeb9f4a078145b883b6874b"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:core5:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/target/httpcore5-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5/target/httpcore5-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2?package-id=a279be19b9d88ebc","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-h2","cpe":"cpe:2.3:a:apache:httpcore5-h2:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_h2:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-h2"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2@5.2.4?package-id=d796643e389f2e06","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-h2","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5-h2:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-h2@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"8738d6dcdf91e66ab1f5f29611bab891442c1334"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_h2:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/target/httpcore5-h2-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-h2"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5-h2/target/httpcore5-h2-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive?package-id=6a13ea19533712fb","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-reactive","cpe":"cpe:2.3:a:apache:httpcore5-reactive:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_reactive:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-reactive"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive@5.2.4?package-id=7611b23e585eea6b","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-reactive","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5-reactive:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-reactive@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"1e1af4da412b6d0642023da1e27e4cd6f7258543"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_reactive:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/target/httpcore5-reactive-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-reactive"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5-reactive/target/httpcore5-reactive-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-testing@5.2.4?package-id=4494f8b4fb836d3d","type":"library","group":"org.apache.httpcomponents.core5","name":"httpcore5-testing","version":"5.2.4","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:apache:httpcore5-testing:5.2.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.httpcomponents.core5/httpcore5-testing@5.2.4","externalReferences":[{"url":"","hashes":[{"alg":"SHA-1","content":"cb560b4e3b193fcb22b595a4264a32eda10e7e04"}],"type":"build-meta"}],"properties":[{"name":"syft:package:foundBy","value":"java-archive-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:httpcore5_testing:5.2.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/target/httpcore5-testing-5.2.4.jar"},{"name":"syft:metadata:-:artifactID","value":"httpcore5-testing"},{"name":"syft:metadata:-:groupID","value":"org.apache.httpcomponents.core5"},{"name":"syft:metadata:virtualPath","value":"/httpcore5-testing/target/httpcore5-testing-5.2.4.jar"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=54fa850ed3b0a738","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=90b7ee5db4cdf400","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=87befb3f5a3892c7","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.junit.jupiter/junit-jupiter?package-id=dedea66dffcbb2ef","type":"library","group":"org.junit.jupiter","name":"junit-jupiter","cpe":"cpe:2.3:a:junit-jupiter:junit-jupiter:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.junit.jupiter/junit-jupiter","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit-jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit_jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit-jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:junit_jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:jupiter:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:junit:jupiter:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"junit-jupiter"},{"name":"syft:metadata:-:groupID","value":"org.junit.jupiter"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-core?package-id=2942099118a1605c","type":"library","group":"org.apache.logging.log4j","name":"log4j-core","cpe":"cpe:2.3:a:apache:log4j-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-core"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-core?package-id=dc37187bca6ec8ba","type":"library","group":"org.apache.logging.log4j","name":"log4j-core","cpe":"cpe:2.3:a:apache:log4j-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-core"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-core?package-id=0392d5abcc91b4a7","type":"library","group":"org.apache.logging.log4j","name":"log4j-core","cpe":"cpe:2.3:a:apache:log4j-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-core"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl?package-id=d369582b74e54c18","type":"library","group":"org.apache.logging.log4j","name":"log4j-slf4j-impl","cpe":"cpe:2.3:a:apache:log4j-slf4j-impl:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_slf4j_impl:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-slf4j-impl"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl?package-id=c9a788cb04f175b3","type":"library","group":"org.apache.logging.log4j","name":"log4j-slf4j-impl","cpe":"cpe:2.3:a:apache:log4j-slf4j-impl:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_slf4j_impl:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-slf4j-impl"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl?package-id=cf1e8c5d0c3675ee","type":"library","group":"org.apache.logging.log4j","name":"log4j-slf4j-impl","cpe":"cpe:2.3:a:apache:log4j-slf4j-impl:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j_slf4j_impl:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"log4j-slf4j-impl"},{"name":"syft:metadata:-:groupID","value":"org.apache.logging.log4j"}]},{"bom-ref":"pkg:maven/org.mockito/mockito-core?package-id=ffb40ca9d59d7ee7","type":"library","group":"org.mockito","name":"mockito-core","cpe":"cpe:2.3:a:mockito-core:mockito-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.mockito/mockito-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito-core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"mockito-core"},{"name":"syft:metadata:-:groupID","value":"org.mockito"}]},{"bom-ref":"pkg:maven/org.mockito/mockito-core?package-id=b8a0324ff25c7d0f","type":"library","group":"org.mockito","name":"mockito-core","cpe":"cpe:2.3:a:mockito-core:mockito-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.mockito/mockito-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito-core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"mockito-core"},{"name":"syft:metadata:-:groupID","value":"org.mockito"}]},{"bom-ref":"pkg:maven/org.mockito/mockito-core?package-id=d6bf1fd483f274a5","type":"library","group":"org.mockito","name":"mockito-core","cpe":"cpe:2.3:a:mockito-core:mockito-core:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.mockito/mockito-core","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito-core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito_core:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito-core:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mockito:mockito_core:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"mockito-core"},{"name":"syft:metadata:-:groupID","value":"org.mockito"}]},{"bom-ref":"pkg:maven/org.reactivestreams/reactive-streams@1.0.4?package-id=9635bc40f8426e4f","type":"library","group":"org.reactivestreams","name":"reactive-streams","version":"1.0.4","cpe":"cpe:2.3:a:reactive-streams:reactive-streams:1.0.4:*:*:*:*:*:*:*","purl":"pkg:maven/org.reactivestreams/reactive-streams@1.0.4","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive-streams:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive_streams:reactive-streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive_streams:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactivestreams:reactive-streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactivestreams:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive:reactive-streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:reactive:reactive_streams:1.0.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"reactive-streams"},{"name":"syft:metadata:-:groupID","value":"org.reactivestreams"}]},{"bom-ref":"pkg:maven/io.reactivex.rxjava2/rxjava@${rxjava.version}?package-id=a39e0831b99e6769","type":"library","group":"io.reactivex.rxjava2","name":"rxjava","version":"${rxjava.version}","cpe":"cpe:2.3:a:reactivex:rxjava:\\$\\{rxjava.version\\}:*:*:*:*:*:*:*","purl":"pkg:maven/io.reactivex.rxjava2/rxjava@${rxjava.version}","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava2:rxjava:\\$\\{rxjava.version\\}:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava:rxjava:\\$\\{rxjava.version\\}:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"rxjava"},{"name":"syft:metadata:-:groupID","value":"io.reactivex.rxjava2"}]},{"bom-ref":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}?package-id=f7333bb2fdb63b9b","type":"library","group":"io.reactivex.rxjava3","name":"rxjava","version":"${rxjava3.version}","cpe":"cpe:2.3:a:reactivex:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*","purl":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava3:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-reactive/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"rxjava"},{"name":"syft:metadata:-:groupID","value":"io.reactivex.rxjava3"}]},{"bom-ref":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}?package-id=948cdeb56f7918be","type":"library","group":"io.reactivex.rxjava3","name":"rxjava","version":"${rxjava3.version}","cpe":"cpe:2.3:a:reactivex:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*","purl":"pkg:maven/io.reactivex.rxjava3/rxjava@${rxjava3.version}","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava3:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rxjava:rxjava:\\$\\{rxjava3.version\\}:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"rxjava"},{"name":"syft:metadata:-:groupID","value":"io.reactivex.rxjava3"}]},{"bom-ref":"pkg:maven/org.slf4j/slf4j-api?package-id=0cd64d32e6da4a4e","type":"library","group":"org.slf4j","name":"slf4j-api","cpe":"cpe:2.3:a:slf4j-api:slf4j-api:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.slf4j/slf4j-api","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j-api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-h2/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"slf4j-api"},{"name":"syft:metadata:-:groupID","value":"org.slf4j"}]},{"bom-ref":"pkg:maven/org.slf4j/slf4j-api?package-id=ce7df01f51b64348","type":"library","group":"org.slf4j","name":"slf4j-api","cpe":"cpe:2.3:a:slf4j-api:slf4j-api:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.slf4j/slf4j-api","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j-api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5-testing/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"slf4j-api"},{"name":"syft:metadata:-:groupID","value":"org.slf4j"}]},{"bom-ref":"pkg:maven/org.slf4j/slf4j-api?package-id=084e13ec4fa8ecbc","type":"library","group":"org.slf4j","name":"slf4j-api","cpe":"cpe:2.3:a:slf4j-api:slf4j-api:*:*:*:*:*:*:*:*","purl":"pkg:maven/org.slf4j/slf4j-api","properties":[{"name":"syft:package:foundBy","value":"java-pom-cataloger"},{"name":"syft:package:language","value":"java"},{"name":"syft:package:type","value":"java-archive"},{"name":"syft:package:metadataType","value":"java-archive"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j-api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j_api:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j-api:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slf4j:slf4j_api:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/httpcore5/pom.xml"},{"name":"syft:metadata:-:artifactID","value":"slf4j-api"},{"name":"syft:metadata:-:groupID","value":"org.slf4j"}]}]} From f01788cc264f16ca9baecf2d4262ef52d4372fbd Mon Sep 17 00:00:00 2001 From: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> Date: Fri, 12 Apr 2024 08:53:53 +0200 Subject: [PATCH 22/42] Code clean: style fixes Signed-off-by: Gijs Calis <51088038+GijsCalis@users.noreply.github.com> --- syft/pkg/cataloger/java/maven_repo_utils.go | 133 ++++++++++---------- syft/pkg/cataloger/java/parse_pom_xml.go | 19 ++- 2 files changed, 72 insertions(+), 80 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index c502fb52450..a2cc46a3717 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -27,8 +27,8 @@ type mavenCoordinate struct { // and also to prevent downloading multiple times from a remote repository. var parsedPomFilesCache map[mavenCoordinate]*gopom.Project = make(map[mavenCoordinate]*gopom.Project) -var checkedForMavenLocalRepo bool = false -var mavenLocalRepoDir string = "" +var checkedForMavenLocalRepo = false +var mavenLocalRepoDir = "" func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (requestURL string, err error) { // groupID needs to go from maven.org -> maven/org @@ -50,7 +50,6 @@ func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (reque // call processedPomFiles should be nil. It is used to prevent cycles (endless loops). func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, processedPomFiles map[mavenCoordinate]bool) string { - // Create map to keep track of processed pom files and to prevent cycles. if processedPomFiles == nil { processedPomFiles = make(map[mavenCoordinate]bool) @@ -63,9 +62,10 @@ func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroup if alreadyProcessed { log.Debug("skipping already processed pom.") return "" - } else { - processedPomFiles[pomCoordinates] = true } + + processedPomFiles[pomCoordinates] = true + addMissingPropertiesFromProject(allProperties, pom) foundDepMngVersion := "" @@ -126,14 +126,12 @@ func isPropertyResolved(value string) bool { // May recursively call recursivelyFindVersionFromManagedOrInherited when a Maven BOM is found. func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, processedPomFiles map[mavenCoordinate]bool) string { - for _, dependency := range *getPomManagedDependencies(pom) { log.Tracef("got managed dependency: [%s, %s, %s]", safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) // imported pom files should be treated just like parent poms, they are use to define versions of dependencies if safeString(dependency.Type) == "pom" && safeString(dependency.Scope) == "import" { - bomVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) log.Debugf("found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) @@ -159,7 +157,6 @@ func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArt } } } - } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { foundVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) if isPropertyResolved(foundVersion) { @@ -181,7 +178,7 @@ func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifact log.Debugf("recursively finding licenses from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", groupID, artifactID, groupID, artifactID, version) var licenses []string - var foundPom bool = false + var foundPom = false processedPomFiles := make(map[mavenCoordinate]bool) // As there can be nested parent poms, we'll recursively check for licenses until no parent is found @@ -208,10 +205,10 @@ func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifact if alreadyProcessed { log.Debug("already processed parent pom, stop searching.") break - } else { - processedPomFiles[pomCoordinates] = true } + processedPomFiles[pomCoordinates] = true + groupID = pomCoordinates.GroupID artifactID = pomCoordinates.ArtifactID version = pomCoordinates.Version @@ -223,7 +220,7 @@ func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifact // Get a parent pom from cache, local repository or download from a Maven repository func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version string, allProperties map[string]string, cfg ArchiveCatalogerConfig) (*gopom.Project, error) { - var err error = nil + var err error if !isPropertyResolved(version) { return nil, fmt.Errorf("cannot get POM without resolved version: %s", version) @@ -234,34 +231,35 @@ func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version st if found { return parentPom, err - } else { - // Then try to get from local file system. - if cfg.UseMavenLocalRepository { - parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version, cfg) - } + } - if !found && cfg.UseNetwork { - // If all fails, then try to get from Maven repository over HTTP - parentPom, err = getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) - if err != nil && parentPom != nil { - found = true - } - } + // Next try to get from local file system (Maven local repository). + if cfg.UseMavenLocalRepository { + parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version, cfg) + } - if found { - // Get and add all properties defined in parent poms to this project for resolving properties later on. - if parentPom.Parent != nil { - getPropertiesFromParentPoms( - ctx, allProperties, *parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version, - ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) - } - addPropertiesToProject(parentPom, allProperties) - addMissingPropertiesFromProject(allProperties, parentPom) + if !found && cfg.UseNetwork { + // If all fails, then try to get from Maven repository over HTTP + parentPom, err = getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) + if err != nil && parentPom != nil { + found = true + } + } - // Store in cache - parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] = parentPom + if found { + // Get and add all properties defined in parent poms to this project for resolving properties later on. + if parentPom.Parent != nil { + getPropertiesFromParentPoms( + ctx, allProperties, *parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version, + ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) } + addPropertiesToProject(parentPom, allProperties) + addMissingPropertiesFromProject(allProperties, parentPom) + + // Store in cache + parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] = parentPom } + return parentPom, err } @@ -290,9 +288,10 @@ func getPomFromMavenUserLocalRepository(groupID, artifactID, version string, cfg return nil, false } return &pom, true - } else { - log.Debugf("could not find pom file: [%s]", pomFile) } + + log.Debugf("could not find pom file: [%s]", pomFile) + return nil, false } @@ -301,22 +300,22 @@ func getLocalRepositoryExists(cfg ArchiveCatalogerConfig) (string, bool) { found := false if checkedForMavenLocalRepo { if mavenLocalRepoDir != "" { + // dir was found in previous call of this function found = true } return mavenLocalRepoDir, found + } + + localRepoDir := cfg.MavenLocalRepositoryDir + if _, err := os.Stat(localRepoDir); !os.IsNotExist(err) { + mavenLocalRepoDir = localRepoDir + found = true } else { - if !checkedForMavenLocalRepo { - localRepoDir := cfg.MavenLocalRepositoryDir - if _, err := os.Stat(localRepoDir); !os.IsNotExist(err) { - mavenLocalRepoDir = localRepoDir - found = true - } else { - log.Errorf("local Maven repository not found at [%s],", localRepoDir) - } - checkedForMavenLocalRepo = true - } - return mavenLocalRepoDir, found + log.Warnf("local Maven repository not found at [%s],", localRepoDir) } + checkedForMavenLocalRepo = true + + return mavenLocalRepoDir, found } // Get default location of the Maven local repository at /.m2/repository @@ -329,9 +328,8 @@ func getDefaultMavenLocalRepoLocation() (string, error) { localRepoDir := filepath.Join(homeDir, ".m2", "repository") if _, err := os.Stat(homeDir); !os.IsNotExist(err) { return localRepoDir, nil - } else { - return "", fmt.Errorf("local Maven repository not found at default location [%s],", localRepoDir) } + return "", fmt.Errorf("local Maven repository not found at default location [%s],", localRepoDir) } // Download the pom file from a (remote) Maven repository over HTTP. @@ -418,14 +416,13 @@ func getPomDependencies(pom *gopom.Project) *[]gopom.Dependency { } } return &dependencies + } - } else { - if pom.Dependencies != nil { - return pom.Dependencies - } else { - return &dependencies - } + if pom.Dependencies != nil { + return pom.Dependencies } + + return &dependencies } // Returns all managed dependencies in a project, including all defined in profiles. @@ -442,15 +439,14 @@ func getPomManagedDependencies(pom *gopom.Project) *[]gopom.Dependency { } } return &mDependencies + } - } else { - if pom.DependencyManagement != nil && pom.DependencyManagement.Dependencies != nil { - return pom.DependencyManagement.Dependencies - } else { - var mDependencies []gopom.Dependency = make([]gopom.Dependency, 0) - return &mDependencies - } + if pom.DependencyManagement != nil && pom.DependencyManagement.Dependencies != nil { + return pom.DependencyManagement.Dependencies } + + var mDependencies []gopom.Dependency = make([]gopom.Dependency, 0) + return &mDependencies } // Traverse the parent pom hierarchy and return all found properties. @@ -459,7 +455,6 @@ func getPomManagedDependencies(pom *gopom.Project) *[]gopom.Dependency { // is found. func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]string, parentGroupID, parentArtifactID, parentVersion string, cfg ArchiveCatalogerConfig, parsedPomFiles map[mavenCoordinate]bool) { - // Create map to keep track of parsed pom files and to prevent cycles. if parsedPomFiles == nil { parsedPomFiles = make(map[mavenCoordinate]bool) @@ -502,9 +497,8 @@ func resolveRecursiveByPropertyName(pomProperties map[string]string, propertyNam if value, ok := pomProperties[name]; ok { if strings.HasPrefix(value, "${") { return resolveRecursiveByPropertyName(pomProperties, value) - } else { - return value } + return value } } return propertyName @@ -523,11 +517,12 @@ func getPropertyName(value string) string { // Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom.Project) { if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { - for name, value := range pom.Properties.Entries { + for name := range pom.Properties.Entries { // Add property from pom that is not yet in allProperties map. _, exists := allProperties[name] if !exists { - value = resolveProperty(*pom, &value, getPropertyName(value)) + currentValue := pom.Properties.Entries[name] + value := resolveProperty(*pom, ¤tValue, getPropertyName(currentValue)) allProperties[name] = value // log.Tracef("added property ['%s'='%s'] from pom [%s, %s, %s] to allProperties", name, value, // *pom.GroupID, *pom.ArtifactID, *pom.Version) @@ -539,14 +534,12 @@ func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom allProperties[name] = resolveRecursiveByPropertyName(allProperties, value) } } - } } // Add all properties from map 'allProperties' to the project 'pom' that are not already defined in the pom. // This increases the chance of the 'resolveProperty' function succeeding. func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) { - if len(allProperties) > 0 { if pom.Properties == nil { var props gopom.Properties diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index fb5cf9195dd..09587185ad4 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -35,7 +35,7 @@ func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file. // Add all properties defined in parent poms to this project for resolving properties later on. if pom.Parent != nil { - var allProperties map[string]string = make(map[string]string) + var allProperties = make(map[string]string) getPropertiesFromParentPoms( ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, gap.cfg, nil) addPropertiesToProject(&pom, allProperties) @@ -72,7 +72,7 @@ func parsePomXMLProject(ctx context.Context, path string, reader io.Reader, loca // Add all properties defined in parent poms to this project for resolving properties later on. if pom.Parent != nil { - var allProperties map[string]string = make(map[string]string) + var allProperties = make(map[string]string) getPropertiesFromParentPoms( ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg, nil) addPropertiesToProject(&pom, allProperties) @@ -122,20 +122,20 @@ func newPomProject(path string, p gopom.Project, location file.Location) *parsed } func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Dependency, cfg ArchiveCatalogerConfig, locations ...file.Location) pkg.Package { - groupId := resolveProperty(pom, dep.GroupID, "groupId") - artifactId := resolveProperty(pom, dep.ArtifactID, "artifactId") + groupID := resolveProperty(pom, dep.GroupID, "groupId") + artifactID := resolveProperty(pom, dep.ArtifactID, "artifactId") m := pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ - GroupID: groupId, - ArtifactID: artifactId, + GroupID: groupID, + ArtifactID: artifactID, Scope: resolveProperty(pom, dep.Scope, "scope"), }, } name := safeString(dep.ArtifactID) version := resolveProperty(pom, dep.Version, "version") - var allProperties map[string]string = make(map[string]string) + var allProperties = make(map[string]string) addMissingPropertiesFromProject(allProperties, &pom) licenses := make([]pkg.License, 0) @@ -153,7 +153,6 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende version = resolveProperty(pom, &version, getPropertyName(version)) } if isPropertyResolved(version) { - parentLicenses, _ := recursivelyFindLicensesFromParentPom( ctx, m.PomProperties.GroupID, @@ -167,7 +166,7 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende } } } else { - log.Warnf("could not determine version for package: [%s, %s]", groupId, artifactId) + log.Warnf("could not determine version for package: [%s, %s]", groupID, artifactID) } if strings.HasPrefix(version, "${") { @@ -285,7 +284,7 @@ func cleanDescription(original *string) (cleaned string) { func resolveProperty(pom gopom.Project, propertyValue *string, propertyName string) string { propertyCase := safeString(propertyValue) if !strings.Contains(propertyCase, "${") { - //nothing to resolve + // nothing to resolve // log.Tracef("resolving property: value [%s] contains no variable", propertyName) return propertyCase } From f941def456d2a14e43cdf695e3c63571a88ced7c Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 03:05:48 -0400 Subject: [PATCH 23/42] chore: initial refactor to use mavenResolver Signed-off-by: Keith Zantow --- cmd/syft/internal/options/catalog.go | 4 +- cmd/syft/internal/options/java.go | 7 +- syft/pkg/cataloger/java/archive_parser.go | 61 +- .../pkg/cataloger/java/archive_parser_test.go | 55 +- syft/pkg/cataloger/java/cataloger.go | 2 +- syft/pkg/cataloger/java/config.go | 12 +- syft/pkg/cataloger/java/maven_repo_utils.go | 597 ++++-------------- syft/pkg/cataloger/java/maven_resolver.go | 188 ++++++ .../pkg/cataloger/java/maven_resolver_test.go | 124 ++++ syft/pkg/cataloger/java/maven_utils.go | 47 ++ syft/pkg/cataloger/java/maven_utils_test.go | 77 +++ syft/pkg/cataloger/java/parse_pom_xml.go | 357 ++++++----- syft/pkg/cataloger/java/parse_pom_xml_test.go | 63 +- .../.m2/settings.xml | 4 + .../org/child-one/1.3.6/child-one-1.3.6.pom | 42 ++ .../org/child-two/2.1.90/child-two-2.1.90.pom | 53 ++ .../parent-one/3.11.0/parent-one-3.11.0.pom | 52 ++ .../parent-two/13.7.8/parent-two-13.7.8.pom | 60 ++ .../parent/7.11.2}/parent-7.11.2.pom | 0 .../junit/junit-bom/5.9.1/junit-bom-5.9.1.pom | 159 +++++ .../3.4.6}/opensaml-parent-3.4.6.pom | 0 21 files changed, 1173 insertions(+), 791 deletions(-) create mode 100644 syft/pkg/cataloger/java/maven_resolver.go create mode 100644 syft/pkg/cataloger/java/maven_resolver_test.go create mode 100644 syft/pkg/cataloger/java/maven_utils.go create mode 100644 syft/pkg/cataloger/java/maven_utils_test.go create mode 100644 syft/pkg/cataloger/java/test-fixtures/local-repository-settings/.m2/settings.xml create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-two/2.1.90/child-two-2.1.90.pom create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom rename syft/pkg/cataloger/java/test-fixtures/{maven-xml-responses => pom/maven-repo/net/shibboleth/parent/7.11.2}/parent-7.11.2.pom (100%) create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom rename syft/pkg/cataloger/java/test-fixtures/{maven-xml-responses => pom/maven-repo/org/opensaml/opensaml-parent/3.4.6}/opensaml-parent-3.4.6.pom (100%) diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 15da1d9c158..99359abec51 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -151,11 +151,11 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements, }, JavaArchive: java.DefaultArchiveCatalogerConfig(). - WithUseNetwork(cfg.Java.UseNetwork). WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository). WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir). + WithUseNetwork(cfg.Java.UseNetwork). WithMavenBaseURL(cfg.Java.MavenURL). - WithArchiveTraversal(archiveSearch), + WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth), } } diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index 26b92b60e6b..979d647df88 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -1,8 +1,9 @@ package options -import "github.com/anchore/clio" - -import "github.com/anchore/syft/syft/pkg/cataloger/java" +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/pkg/cataloger/java" +) type javaConfig struct { UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 9bdfd5f0112..6519ab530f0 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -49,6 +49,7 @@ type archiveParser struct { fileInfo archiveFilename detectNested bool cfg ArchiveCatalogerConfig + mavenResolver } type genericArchiveParserAdapter struct { @@ -96,13 +97,14 @@ func newJavaArchiveParser(reader file.LocationReadCloser, detectNested bool, cfg } return &archiveParser{ - fileManifest: fileManifest, - location: reader.Location, - archivePath: archivePath, - contentPath: contentPath, - fileInfo: newJavaArchiveFilename(currentFilepath), - detectNested: detectNested, - cfg: cfg, + fileManifest: fileManifest, + location: reader.Location, + archivePath: archivePath, + contentPath: contentPath, + fileInfo: newJavaArchiveFilename(currentFilepath), + detectNested: detectNested, + cfg: cfg, + mavenResolver: newMavenResolver(cfg), }, cleanupFn, nil } @@ -194,7 +196,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, return nil, err } - licenses, name, version, err := j.parseLicenses(ctx, manifest) + name, version, licenses, err := j.discoverNameVersionLicense(ctx, manifest) if err != nil { return nil, err } @@ -217,7 +219,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, }, nil } -func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaManifest) ([]pkg.License, string, string, error) { +func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest *pkg.JavaManifest) (string, string, []pkg.License, error) { // we use j.location because we want to associate the license declaration with where we discovered the contents in the manifest // TODO: when we support locations of paths within archives we should start passing the specific manifest location object instead of the top jar licenses := pkg.NewLicensesFromLocation(j.location, selectLicenses(manifest)...) @@ -228,7 +230,7 @@ func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaMan 3. manifest 4. filename */ - group, name, version, pomLicenses := j.guessMainPackageNameAndVersionFromPomInfo(ctx, j.cfg) + group, name, version, pomLicenses := j.discoverMainPackageNameAndVersionFromPomInfo(ctx) if name == "" { name = selectName(manifest, j.fileInfo) } @@ -245,7 +247,7 @@ func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaMan if len(licenses) == 0 { fileLicenses, err := j.getLicenseFromFileInArchive() if err != nil { - return nil, "", "", err + return "", "", nil, err } if fileLicenses != nil { licenses = append(licenses, fileLicenses...) @@ -254,13 +256,13 @@ func (j *archiveParser) parseLicenses(ctx context.Context, manifest *pkg.JavaMan // If we didn't find any licenses in the archive so far, we'll try again in Maven Central using groupIDFromJavaMetadata if len(licenses) == 0 && j.cfg.UseNetwork { - licenses = findLicenseFromJavaMetadata(ctx, group, name, manifest, version, j, licenses) + licenses = j.findLicenseFromJavaMetadata(ctx, group, name, manifest, version, licenses) } - return licenses, name, version, nil + return name, version, licenses, nil } -func findLicenseFromJavaMetadata(ctx context.Context, group, name string, manifest *pkg.JavaManifest, version string, j *archiveParser, licenses []pkg.License) []pkg.License { +func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, group, name string, manifest *pkg.JavaManifest, version string, licenses []pkg.License) []pkg.License { var groupID = group if group == "" { if gID := groupIDFromJavaMetadata(name, pkg.JavaArchive{Manifest: manifest}); gID != "" { @@ -268,15 +270,13 @@ func findLicenseFromJavaMetadata(ctx context.Context, group, name string, manife } } - pomLicenses, foundPom := recursivelyFindLicensesFromParentPom(ctx, groupID, name, version, j.cfg) - log.Tracef("[REMOVE] 1 findLicenseFromJavaMetadata, coordinates %s:%s:%s", groupID, name, version) + pomLicenses, err := j.resolveLicenses(ctx, groupID, name, version) - if !foundPom && len(pomLicenses) == 0 { + if err == nil && len(pomLicenses) == 0 { // Try removing the last part of the groupId, as sometimes it duplicates the artifactId packages := strings.Split(groupID, ".") groupID = strings.Join(packages[:len(packages)-1], ".") - log.Tracef("[REMOVE] 2 findLicenseFromJavaMetadata, coordinates %s:%s:%s", groupID, name, version) - pomLicenses, _ = recursivelyFindLicensesFromParentPom(ctx, groupID, name, version, j.cfg) + pomLicenses, _ = j.resolveLicenses(ctx, groupID, name, version) } if len(pomLicenses) > 0 { @@ -294,7 +294,7 @@ type parsedPomProject struct { } // Try to find artifact group, id, version and license(s) -func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Context, cfg ArchiveCatalogerConfig) (group, name, version string, licenses []pkg.License) { +func (j *archiveParser) discoverMainPackageNameAndVersionFromPomInfo(ctx context.Context) (group, name, version string, licenses []pkg.License) { pomPropertyMatches := j.fileManifest.GlobMatch(false, pomPropertiesGlob) pomMatches := j.fileManifest.GlobMatch(false, pomXMLGlob) var pomPropertiesObject pkg.JavaPomProperties @@ -302,7 +302,7 @@ func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Co // Find the pom.properties/pom.xml if the names seem like a plausible match properties, _ := pomPropertiesByParentPath(j.archivePath, j.location, pomPropertyMatches) - projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, pomMatches, cfg) + projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, pomMatches, j.cfg) for parentPath, propertiesObj := range properties { if artifactIDMatchesFilename(propertiesObj.ArtifactID, j.fileInfo.name) { @@ -329,14 +329,14 @@ func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Co if pomProjectObject == nil { // If we have no pom.xml, check maven central using pom.properties - parentLicenses, _ := recursivelyFindLicensesFromParentPom(ctx, pomPropertiesObject.GroupID, pomPropertiesObject.ArtifactID, pomPropertiesObject.Version, j.cfg) + parentLicenses, _ := j.resolveLicenses(ctx, pomPropertiesObject.GroupID, pomPropertiesObject.ArtifactID, pomPropertiesObject.Version) if len(parentLicenses) > 0 { for _, licenseName := range parentLicenses { licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } } } else { - findPomLicenses(ctx, pomProjectObject, j.cfg) + findPomLicenses(ctx, pomProjectObject, &j.mavenResolver) } if pomProjectObject != nil { @@ -353,15 +353,14 @@ func artifactIDMatchesFilename(artifactID, fileName string) bool { return strings.HasPrefix(artifactID, fileName) || strings.HasSuffix(fileName, artifactID) } -func findPomLicenses(ctx context.Context, pomProjectObject *parsedPomProject, cfg ArchiveCatalogerConfig) { +func findPomLicenses(ctx context.Context, pomProjectObject *parsedPomProject, r *mavenResolver) { // If we don't have any licenses until now, and if we have a parent Pom, then we'll check the parent pom in maven central for licenses. if pomProjectObject != nil && pomProjectObject.Parent != nil && len(pomProjectObject.Licenses) == 0 { - parentLicenses, _ := recursivelyFindLicensesFromParentPom( + parentLicenses, _ := r.resolveLicenses( ctx, pomProjectObject.Parent.GroupID, pomProjectObject.Parent.ArtifactID, - pomProjectObject.Parent.Version, - cfg) + pomProjectObject.Parent.Version) if len(parentLicenses) > 0 { for _, licenseName := range parentLicenses { @@ -400,7 +399,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren pomProject = proj } - pkgFromPom := newPackageFromMavenData(ctx, propertiesObj, pomProject, parentPkg, j.location, j.cfg) + pkgFromPom := newPackageFromMavenData(ctx, propertiesObj, pomProject, parentPkg, j.location, &j.mavenResolver) if pkgFromPom != nil { pkgs = append(pkgs, *pkgFromPom) } @@ -591,7 +590,7 @@ func pomProjectByParentPath(ctx context.Context, archivePath string, location fi // newPackageFromMavenData processes a single Maven POM properties for a given parent package, returning all listed Java packages found and // associating each discovered package to the given parent package. Note the pom.xml is optional, the pom.properties is not. -func newPackageFromMavenData(ctx context.Context, pomProperties pkg.JavaPomProperties, parsedPomProject *parsedPomProject, parentPkg *pkg.Package, location file.Location, cfg ArchiveCatalogerConfig) *pkg.Package { +func newPackageFromMavenData(ctx context.Context, pomProperties pkg.JavaPomProperties, parsedPomProject *parsedPomProject, parentPkg *pkg.Package, location file.Location, r *mavenResolver) *pkg.Package { // keep the artifact name within the virtual path if this package does not match the parent package vPathSuffix := "" groupID := "" @@ -618,14 +617,14 @@ func newPackageFromMavenData(ctx context.Context, pomProperties pkg.JavaPomPrope if parsedPomProject == nil { // If we have no pom.xml, check maven central using pom.properties - parentLicenses, _ := recursivelyFindLicensesFromParentPom(ctx, pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version, cfg) + parentLicenses, _ := r.resolveLicenses(ctx, pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version) if len(parentLicenses) > 0 { for _, licenseName := range parentLicenses { licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } } } else { - findPomLicenses(ctx, parsedPomProject, cfg) + findPomLicenses(ctx, parsedPomProject, r) } if parsedPomProject != nil { diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index c463bc323bf..5c1870c3cdc 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "net/http/httptest" "os" "os/exec" "path/filepath" @@ -68,21 +67,14 @@ func generateMockMavenHandler(responseFixture string) func(w http.ResponseWriter } } -type handlerPath struct { - path string - handler func(w http.ResponseWriter, r *http.Request) -} - func TestSearchMavenForLicenses(t *testing.T) { - mux, url, teardown := setup() - defer teardown() + url := testRepo(t, "test-fixtures/pom/maven-repo") + tests := []struct { name string fixture string detectNested bool config ArchiveCatalogerConfig - requestPath string - requestHandlers []handlerPath expectedLicenses []pkg.License }{ { @@ -93,16 +85,7 @@ func TestSearchMavenForLicenses(t *testing.T) { UseNetwork: true, UseMavenLocalRepository: false, MavenBaseURL: url, - }, - requestHandlers: []handlerPath{ - { - path: "/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom", - handler: generateMockMavenHandler("test-fixtures/maven-xml-responses/opensaml-parent-3.4.6.pom"), - }, - { - path: "/net/shibboleth/parent/7.11.2/parent-7.11.2.pom", - handler: generateMockMavenHandler("test-fixtures/maven-xml-responses/parent-7.11.2.pom"), - }, + MaxParentRecursiveDepth: 5, }, expectedLicenses: []pkg.License{ { @@ -116,11 +99,6 @@ func TestSearchMavenForLicenses(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - // configure maven central requests - for _, hdlr := range tc.requestHandlers { - mux.HandleFunc(hdlr.path, hdlr.handler) - } - // setup metadata fixture; note: // this fixture has a pomProjectObject and has a parent object // it has no licenses on either which is the condition for testing @@ -138,13 +116,13 @@ func TestSearchMavenForLicenses(t *testing.T) { defer cleanupFn() // assert licenses are discovered from upstream - _, _, _, licenses := ap.guessMainPackageNameAndVersionFromPomInfo(context.Background(), tc.config) + _, _, _, licenses := ap.discoverMainPackageNameAndVersionFromPomInfo(context.Background()) assert.Equal(t, tc.expectedLicenses, licenses) }) } } -func TestFormatMavenURL(t *testing.T) { +func Test_remotePomURL(t *testing.T) { tests := []struct { name string groupID string @@ -163,7 +141,7 @@ func TestFormatMavenURL(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - requestURL, err := formatMavenPomURL(tc.groupID, tc.artifactID, tc.version, mavenBaseURL) + requestURL, err := remotePomURL(mavenBaseURL, tc.groupID, tc.artifactID, tc.version) assert.NoError(t, err, "expected no err; got %w", err) assert.Equal(t, tc.expected, requestURL) }) @@ -1126,7 +1104,8 @@ func Test_newPackageFromMavenData(t *testing.T) { } test.expectedParent.Locations = locations - actualPackage := newPackageFromMavenData(context.Background(), test.props, test.project, test.parent, file.NewLocation(virtualPath), DefaultArchiveCatalogerConfig()) + r := newMavenResolver(DefaultArchiveCatalogerConfig()) + actualPackage := newPackageFromMavenData(context.Background(), test.props, test.project, test.parent, file.NewLocation(virtualPath), &r) if test.expectedPackage == nil { require.Nil(t, actualPackage) } else { @@ -1471,21 +1450,3 @@ func run(t testing.TB, cmd *exec.Cmd) { } } } - -// setup sets up a test HTTP server for mocking requests to maven central. -// The returned url is injected into the Config so the client uses the test server. -// Tests should register handlers on mux to simulate the expected request/response structure -func setup() (mux *http.ServeMux, serverURL string, teardown func()) { - // mux is the HTTP request multiplexer used with the test server. - mux = http.NewServeMux() - - // We want to ensure that tests catch mistakes where the endpoint URL is - // specified as absolute rather than relative. It only makes a difference - // when there's a non-empty base URL path. So, use that. See issue #752. - apiHandler := http.NewServeMux() - apiHandler.Handle("/", mux) - // server is a test HTTP server used to provide mock API responses. - server := httptest.NewServer(apiHandler) - - return mux, server.URL, server.Close -} diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 9552b142ddf..35f4cbdb2e9 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -35,7 +35,7 @@ func NewPomCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { gap := newGenericArchiveParserAdapter(cfg) return generic.NewCataloger("java-pom-cataloger"). - WithParserByGlobs(gap.parserPomXML, "**/pom.xml") + WithParserByGlobs(gap.parsePomXML, "**/pom.xml") } // NewGradleLockfileCataloger returns a cataloger capable of parsing dependencies from a gradle.lockfile file. diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index c447637fc43..63566c1d6fa 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -7,19 +7,18 @@ const mavenBaseURL = "https://repo1.maven.org/maven2" type ArchiveCatalogerConfig struct { cataloging.ArchiveSearchConfig `yaml:",inline" json:"" mapstructure:",squash"` UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` - UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepositoryk"` + UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` MavenBaseURL string `yaml:"maven-base-url" json:"maven-base-url" mapstructure:"maven-base-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { - localRepoDir, _ := getDefaultMavenLocalRepoLocation() return ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.DefaultArchiveSearchConfig(), UseNetwork: false, - UseMavenLocalRepository: true, - MavenLocalRepositoryDir: localRepoDir, + UseMavenLocalRepository: false, + MavenLocalRepositoryDir: defaultMavenLocalRepoDir(), MavenBaseURL: mavenBaseURL, MaxParentRecursiveDepth: 5, } @@ -47,7 +46,10 @@ func (j ArchiveCatalogerConfig) WithMavenBaseURL(input string) ArchiveCatalogerC return j } -func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSearchConfig) ArchiveCatalogerConfig { +func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSearchConfig, maxDepth int) ArchiveCatalogerConfig { + if maxDepth > 0 { + j.MaxParentRecursiveDepth = maxDepth + } j.ArchiveSearchConfig = search return j } diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go index a2cc46a3717..2ae86ef9018 100644 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -3,555 +3,192 @@ package java import ( "context" "fmt" - "io" - "net/http" "net/url" - "os" - "path/filepath" + "slices" "strings" - "time" "github.com/vifraa/gopom" - - "github.com/anchore/syft/internal/log" ) -// mavenCoordinate is the unique identifier for a package in Maven. -type mavenCoordinate struct { - GroupID string - ArtifactID string - Version string -} - -// Map containing all pom.xml files that have been parsed. They are cached because properties might have been added -// and also to prevent downloading multiple times from a remote repository. -var parsedPomFilesCache map[mavenCoordinate]*gopom.Project = make(map[mavenCoordinate]*gopom.Project) - -var checkedForMavenLocalRepo = false -var mavenLocalRepoDir = "" - -func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (requestURL string, err error) { +func remotePomURL(repoURL, groupID, artifactID, version string) (requestURL string, err error) { // groupID needs to go from maven.org -> maven/org urlPath := strings.Split(groupID, ".") artifactPom := fmt.Sprintf("%s-%s.pom", artifactID, version) urlPath = append(urlPath, artifactID, version, artifactPom) - // ex:"https://repo1.maven.org/maven2/groupID/artifactID/artifactPom - requestURL, err = url.JoinPath(mavenBaseURL, urlPath...) + // ex: https://repo1.maven.org/maven2/groupID/artifactID/artifactPom + requestURL, err = url.JoinPath(repoURL, urlPath...) if err != nil { return requestURL, fmt.Errorf("could not construct maven url: %w", err) } return requestURL, err } -// Try to find the version of a dependency (groupID, artifactID) by parsing all parent poms and imported managed dependencies (maven BOMs). -// Properties are gathered in the order that they are encountered: in Maven the latest definition of a property (highest in hierarchy) is used. -// processedPomFiles contains all previously processed pom files encountered by earlier invocations of this function on the stack. So for the first -// call processedPomFiles should be nil. It is used to prevent cycles (endless loops). -func recursivelyFindVersionFromManagedOrInherited(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, processedPomFiles map[mavenCoordinate]bool) string { - // Create map to keep track of processed pom files and to prevent cycles. - if processedPomFiles == nil { - processedPomFiles = make(map[mavenCoordinate]bool) - } - log.Debugf("recursively finding version from managed or inherited dependencies for dependency [%v:%v] in pom [%s, %s, %s]. recursion depth: %d", - findGroupID, findArtifactID, *pom.GroupID, *pom.ArtifactID, *pom.Version, len(processedPomFiles)) - - pomCoordinates := mavenCoordinate{*pom.GroupID, *pom.ArtifactID, *pom.Version} - _, alreadyProcessed := processedPomFiles[pomCoordinates] - if alreadyProcessed { - log.Debug("skipping already processed pom.") - return "" - } - - processedPomFiles[pomCoordinates] = true - - addMissingPropertiesFromProject(allProperties, pom) - - foundDepMngVersion := "" - if pom.DependencyManagement != nil { - foundDepMngVersion = findVersionInDependencyManagement( - ctx, findGroupID, findArtifactID, pom, cfg, allProperties, processedPomFiles) +// Try to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies +// +//nolint:gocognit +func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Project, pom *gopom.Project, groupID, artifactID string, resolving ...mavenID) (string, error) { + id := newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) + if len(resolving) >= r.cfg.MaxParentRecursiveDepth { + return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) } - if isPropertyResolved(foundDepMngVersion) { - return foundDepMngVersion + if slices.Contains(resolving, id) { + return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) } + resolving = append(resolving, id) - // If a parent exists, search it recursively. - foundRecVersion := "" - if pom.Parent != nil { - parentGroupID := *pom.Parent.GroupID - parentArtifactID := *pom.Parent.ArtifactID - parentVersion := *pom.Parent.Version - - parentPom, err := getPomFromCacheOrMaven(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) - - if parentPom != nil { - log.Debugf("found a parent pom: [%s, %s, %s]", *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version) - foundRecVersion = recursivelyFindVersionFromManagedOrInherited( - ctx, findGroupID, findArtifactID, parentPom, cfg, allProperties, processedPomFiles) - addMissingPropertiesFromProject(allProperties, pom) - addPropertiesToProject(pom, allProperties) - } else { - log.Warnf("unable to get parent pom [%s, %s, %s]: %v", - parentGroupID, parentArtifactID, parentVersion, err) - } - } - - foundVersion := resolveProperty(*pom, &foundRecVersion, getPropertyName(foundRecVersion)) - - // foundDepMngVersion may contain the version in a property that could not previously be resolved. - if !isPropertyResolved(foundVersion) && foundDepMngVersion != "" { - foundDepMngVersion = resolveProperty(*pom, &foundDepMngVersion, getPropertyName(foundDepMngVersion)) - - if isPropertyResolved(foundDepMngVersion) { - foundVersion = foundDepMngVersion + var err error + var version string + + // check for entries in dependencyManagement first + for _, dep := range directManagedDependencies(pom) { + depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) + depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) + if depGroupID == groupID && depArtifactID == artifactID { + version = r.getPropertyValue(ctx, root, dep.Version) + if version != "" { + return version, nil + } } - } - - if foundVersion == "" { - log.Tracef("no version found for dependency: [%s, %s]", findGroupID, findArtifactID) - } else { - log.Debugf("found version [%s] for dependency: [%s, %s]", foundVersion, findGroupID, findArtifactID) - } - return foundVersion -} - -// Returns true when value is not empty and does not start with "${" (contains an unresolved property). -func isPropertyResolved(value string) bool { - return value != "" && !strings.HasPrefix(value, "${") -} -// Find given dependency (groupID, artifactID) in the dependencyManagement section of project 'pom'. -// May recursively call recursivelyFindVersionFromManagedOrInherited when a Maven BOM is found. -func findVersionInDependencyManagement(ctx context.Context, findGroupID, findArtifactID string, - pom *gopom.Project, cfg ArchiveCatalogerConfig, allProperties map[string]string, processedPomFiles map[mavenCoordinate]bool) string { - for _, dependency := range *getPomManagedDependencies(pom) { - log.Tracef("got managed dependency: [%s, %s, %s]", - safeString(dependency.GroupID), safeString(dependency.ArtifactID), safeString(dependency.Version)) + // imported pom files should be treated just like parent poms, they are used to define versions of dependencies + if deref(dep.Type) == "pom" && deref(dep.Scope) == "import" { + depVersion := r.getPropertyValue(ctx, root, dep.Version) - // imported pom files should be treated just like parent poms, they are use to define versions of dependencies - if safeString(dependency.Type) == "pom" && safeString(dependency.Scope) == "import" { - bomVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - log.Debugf("found BOM: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion) - - // Recurse into BOM, which should be treated just like a parent pom - bomProject, err := getPomFromCacheOrMaven(ctx, *dependency.GroupID, *dependency.ArtifactID, bomVersion, allProperties, cfg) - - if err == nil && bomProject != nil { - foundVersion := recursivelyFindVersionFromManagedOrInherited( - ctx, findGroupID, findArtifactID, bomProject, cfg, allProperties, processedPomFiles) - - log.Tracef("finished processing BOM: [%s, %s, %s], found version: [%s]", *dependency.GroupID, *dependency.ArtifactID, bomVersion, foundVersion) - - addMissingPropertiesFromProject(allProperties, pom) - - if isPropertyResolved(foundVersion) { - return foundVersion - } - if foundVersion != "" { - foundVersion = resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - if isPropertyResolved(foundVersion) { - log.Debugf("found version for managed dependency in BOM: [%s, %s, %s]", findGroupID, findArtifactID, foundVersion) - return foundVersion - } - } + depPom, err := r.findPom(ctx, depGroupID, depArtifactID, depVersion) + if err != nil { + return "", err } - } else if *dependency.GroupID == findGroupID && *dependency.ArtifactID == findArtifactID { - foundVersion := resolveProperty(*pom, dependency.Version, getPropertyName(*dependency.Version)) - if isPropertyResolved(foundVersion) { - log.Debugf("found version for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) - return foundVersion + version, err = r.findInheritedVersion(ctx, root, depPom, groupID, artifactID, resolving...) + if err != nil { + return "", err } - if strings.HasPrefix(foundVersion, "${") { - log.Tracef("found version in property reference for managed dependency: [%s, %s, %s]", *dependency.GroupID, *dependency.ArtifactID, foundVersion) - return foundVersion + if version != "" { + return version, nil } } } - log.Tracef("dependency not found in dependencyManagement") - return "" -} - -// Search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. -func recursivelyFindLicensesFromParentPom(ctx context.Context, groupID, artifactID, version string, cfg ArchiveCatalogerConfig) ([]string, bool) { - log.Debugf("recursively finding licenses from parent Pom for artifact [%v:%v], using parent pom: [%v:%v:%v]", - groupID, artifactID, groupID, artifactID, version) - var licenses []string - var foundPom = false - processedPomFiles := make(map[mavenCoordinate]bool) - // As there can be nested parent poms, we'll recursively check for licenses until no parent is found - recursionLevel := 0 - for safeString(&artifactID) != "" { - log.Tracef("recursively find licenses for [%s, %s, %s], recursion level: %d", groupID, artifactID, version, recursionLevel) - recursionLevel++ - - parentPom, err := getPomFromCacheOrMaven(ctx, groupID, artifactID, version, make(map[string]string), cfg) - if err != nil { - // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - log.Tracef("unable to get parent pom: %v", err) - return []string{}, foundPom - } - parentLicenses := parseLicensesFromPom(parentPom) - if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { - licenses = parentLicenses - foundPom = true - break - } - // Check for cycle and store processed pom ID when not - pomCoordinates := mavenCoordinate{*parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version} - _, alreadyProcessed := processedPomFiles[pomCoordinates] - if alreadyProcessed { - log.Debug("already processed parent pom, stop searching.") - break - } - - processedPomFiles[pomCoordinates] = true - - groupID = pomCoordinates.GroupID - artifactID = pomCoordinates.ArtifactID - version = pomCoordinates.Version - } - - return licenses, foundPom -} - -// Get a parent pom from cache, local repository or download from a Maven repository -func getPomFromCacheOrMaven(ctx context.Context, groupID, artifactID, version string, allProperties map[string]string, - cfg ArchiveCatalogerConfig) (*gopom.Project, error) { - var err error - - if !isPropertyResolved(version) { - return nil, fmt.Errorf("cannot get POM without resolved version: %s", version) - } - - // Try get from cache first. - parentPom, found := parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] - - if found { - return parentPom, err - } - - // Next try to get from local file system (Maven local repository). - if cfg.UseMavenLocalRepository { - parentPom, found = getPomFromMavenUserLocalRepository(groupID, artifactID, version, cfg) - } - - if !found && cfg.UseNetwork { - // If all fails, then try to get from Maven repository over HTTP - parentPom, err = getPomFromMavenRepo(ctx, groupID, artifactID, version, cfg.MavenBaseURL) - if err != nil && parentPom != nil { - found = true - } - } - - if found { - // Get and add all properties defined in parent poms to this project for resolving properties later on. - if parentPom.Parent != nil { - getPropertiesFromParentPoms( - ctx, allProperties, *parentPom.Parent.GroupID, *parentPom.Parent.ArtifactID, *parentPom.Parent.Version, - ArchiveCatalogerConfig{MavenBaseURL: mavenBaseURL}, nil) - } - addPropertiesToProject(parentPom, allProperties) - addMissingPropertiesFromProject(allProperties, parentPom) - - // Store in cache - parsedPomFilesCache[mavenCoordinate{groupID, artifactID, version}] = parentPom - } - - return parentPom, err -} - -// Try to get the Pom from the users local repository in the users home dir. -// Returns (nil, false) when file cannot be found or read for any reason. -func getPomFromMavenUserLocalRepository(groupID, artifactID, version string, cfg ArchiveCatalogerConfig) (*gopom.Project, bool) { - localRepoDir, exists := getLocalRepositoryExists(cfg) - - if !exists { - return nil, false + // recursively check parents + parent, err := r.findParent(ctx, pom) + if err != nil { + return "", err } - - groupPath := filepath.Join(strings.Split(groupID, ".")...) - pomFile := filepath.Join(localRepoDir, groupPath, artifactID, version, artifactID+"-"+version+".pom") - - if _, err := os.Stat(pomFile); !os.IsNotExist(err) { - log.Debugf("found pom file: %s", pomFile) - bytes, err := os.ReadFile(pomFile) + if parent != nil { + version, err = r.findInheritedVersion(ctx, root, parent, groupID, artifactID, resolving...) if err != nil { - log.Errorf("could not read pom file: [%s], error: %w", pomFile, err) - return nil, false + return "", err } - pom, err := decodePomXML(strings.NewReader(string(bytes))) - if err != nil { - log.Errorf("could not parse pom file: [%s], error: %w", pomFile, err) - return nil, false + if version != "" { + return version, nil } - return &pom, true } - log.Debugf("could not find pom file: [%s]", pomFile) - - return nil, false -} - -// Get Maven local repository of current user, if it exists. Only checks once and store the result in 'mavenLocalRepoDir'. -func getLocalRepositoryExists(cfg ArchiveCatalogerConfig) (string, bool) { - found := false - if checkedForMavenLocalRepo { - if mavenLocalRepoDir != "" { - // dir was found in previous call of this function - found = true + // check for inherited dependencies + for _, dep := range directDependencies(pom) { + depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) + depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) + if depGroupID == groupID && depArtifactID == artifactID { + version = r.getPropertyValue(ctx, root, dep.Version) + if version != "" { + return version, nil + } } - return mavenLocalRepoDir, found } - localRepoDir := cfg.MavenLocalRepositoryDir - if _, err := os.Stat(localRepoDir); !os.IsNotExist(err) { - mavenLocalRepoDir = localRepoDir - found = true - } else { - log.Warnf("local Maven repository not found at [%s],", localRepoDir) - } - checkedForMavenLocalRepo = true - - return mavenLocalRepoDir, found + return "", nil } -// Get default location of the Maven local repository at /.m2/repository -func getDefaultMavenLocalRepoLocation() (string, error) { - homeDir, err := os.UserHomeDir() - - if err != nil { - return "", err - } - localRepoDir := filepath.Join(homeDir, ".m2", "repository") - if _, err := os.Stat(homeDir); !os.IsNotExist(err) { - return localRepoDir, nil +// resolveLicenses search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. +func (r *mavenResolver) resolveLicenses(ctx context.Context, groupID, artifactID, version string) ([]string, error) { + pom, err := r.findPom(ctx, groupID, artifactID, version) + if pom == nil || err != nil { + return nil, err } - return "", fmt.Errorf("local Maven repository not found at default location [%s],", localRepoDir) + return r.findLicenses(ctx, pom) } -// Download the pom file from a (remote) Maven repository over HTTP. -func getPomFromMavenRepo(ctx context.Context, groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { - if len(groupID) == 0 || len(artifactID) == 0 || !isPropertyResolved(version) { - return nil, fmt.Errorf("missing/incomplete maven artifact coordinates: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) +// Search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. +func (r *mavenResolver) findLicenses(ctx context.Context, pom *gopom.Project, processing ...mavenID) ([]string, error) { + id := makeID(pom) + if slices.Contains(processing, id) { + return nil, fmt.Errorf("cycle detected resolving licenses for: %v", id) } - // Downloading snapshots requires additional steps to determine the latest snapshot version. - // See: https://maven.apache.org/ref/3-LATEST/maven-repository-metadata/ - if strings.HasSuffix(version, "-SNAPSHOT") { - return nil, fmt.Errorf("downloading snapshot artifacts is not supported: groupId:artifactId:version = %s:%s:%s", groupID, artifactID, version) + if len(processing) > r.cfg.MaxParentRecursiveDepth { + return nil, fmt.Errorf("maximum parent recursive depth (%v) reached: %v", r.cfg.MaxParentRecursiveDepth, processing) } - requestURL, err := formatMavenPomURL(groupID, artifactID, version, mavenBaseURL) - if err != nil { - return nil, err + licenses := directLicenses(pom) + if len(licenses) > 0 { + return licenses, nil } - log.Tracef("trying to fetch parent pom from Maven repository %s", requestURL) - mavenRequest, err := http.NewRequest(http.MethodGet, requestURL, nil) + parent, err := r.findParent(ctx, pom) if err != nil { - return nil, fmt.Errorf("unable to format request for Maven central: %w", err) - } - - httpClient := &http.Client{ - Timeout: time.Second * 10, - } - - mavenRequest = mavenRequest.WithContext(ctx) - - resp, err := httpClient.Do(mavenRequest) - if err != nil { - return nil, fmt.Errorf("unable to get pom from Maven repository: %w", err) + return nil, err } - if resp.StatusCode == 404 { - return nil, fmt.Errorf("pom not found in Maven repository") + if parent == nil { + return nil, nil } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Errorf("unable to close body: %+v", err) - } - }() + return r.findLicenses(ctx, parent, append(processing, id)...) +} - bytes, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) - } +// func pomDescriptions(ids []mavenID) []string { +// var out []string +// for _, id := range ids { +// out = append(out, id.String()) +// } +// return out +//} - pom, err := decodePomXML(strings.NewReader(string(bytes))) - if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven repository: %w", err) +func makeID(pom *gopom.Project) mavenID { + if pom == nil { + return mavenID{} } - log.Tracef("fetched parent pom from Maven repository.") - return &pom, nil + return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) } -func parseLicensesFromPom(pom *gopom.Project) []string { +// directLicenses returns the licenses defined directly in the pom +func directLicenses(pom *gopom.Project) []string { var licenses []string - if pom != nil && pom.Licenses != nil { - for _, license := range *pom.Licenses { - if license.Name != nil { - licenses = append(licenses, *license.Name) - } else if license.URL != nil { - licenses = append(licenses, *license.URL) - } + for _, license := range deref(pom.Licenses) { + if license.Name != nil { + licenses = append(licenses, *license.Name) + } else if license.URL != nil { + licenses = append(licenses, *license.URL) } } - return licenses } -// Returns all dependencies in a project, including all defined in profiles. -func getPomDependencies(pom *gopom.Project) *[]gopom.Dependency { - var dependencies []gopom.Dependency = make([]gopom.Dependency, 0) - if pom.Profiles != nil && len(*pom.Profiles) > 0 { - // Gather dependencies from profiles and main dependencies - if pom.Dependencies != nil { - dependencies = append(dependencies, *pom.Dependencies...) - } - - for _, profile := range *pom.Profiles { - if profile.Dependencies != nil { - dependencies = append(dependencies, *profile.Dependencies...) - } - } - return &dependencies - } - - if pom.Dependencies != nil { - return pom.Dependencies - } - - return &dependencies -} - -// Returns all managed dependencies in a project, including all defined in profiles. -func getPomManagedDependencies(pom *gopom.Project) *[]gopom.Dependency { - if pom.Profiles != nil && len(*pom.Profiles) > 0 { - var mDependencies []gopom.Dependency = make([]gopom.Dependency, 0) - if pom.DependencyManagement != nil && pom.DependencyManagement.Dependencies != nil { - mDependencies = append(mDependencies, *pom.DependencyManagement.Dependencies...) - } - - for _, profile := range *pom.Profiles { - if profile.DependencyManagement != nil && profile.DependencyManagement.Dependencies != nil { - mDependencies = append(mDependencies, *profile.DependencyManagement.Dependencies...) - } - } - return &mDependencies - } - - if pom.DependencyManagement != nil && pom.DependencyManagement.Dependencies != nil { - return pom.DependencyManagement.Dependencies +// directDependencies returns all direct dependencies in a project, including all defined in profiles +func directDependencies(pom *gopom.Project) []gopom.Dependency { + dependencies := deref(pom.Dependencies) + for _, profile := range deref(pom.Profiles) { + dependencies = append(dependencies, deref(profile.Dependencies)...) } - - var mDependencies []gopom.Dependency = make([]gopom.Dependency, 0) - return &mDependencies + return dependencies } -// Traverse the parent pom hierarchy and return all found properties. -// To be used for resolving property references later on while determining versions. -// This function recursively processes each encountered parent pom until no parent pom -// is found. -func getPropertiesFromParentPoms(ctx context.Context, allProperties map[string]string, parentGroupID, parentArtifactID, parentVersion string, - cfg ArchiveCatalogerConfig, parsedPomFiles map[mavenCoordinate]bool) { - // Create map to keep track of parsed pom files and to prevent cycles. - if parsedPomFiles == nil { - parsedPomFiles = make(map[mavenCoordinate]bool) - } - log.Debugf("recursively gathering all properties from pom [%s, %s, %s], recursion depth: %d", - parentGroupID, parentArtifactID, parentVersion, len(parsedPomFiles)) - - pomCoordinates := mavenCoordinate{parentGroupID, parentArtifactID, parentVersion} - _, alreadyParsed := parsedPomFiles[pomCoordinates] - if alreadyParsed { - // Nothing new here, already parsed - log.Debug("skipping already processed pom.") - return - } - - parentPom, err := getPomFromCacheOrMaven(ctx, parentGroupID, parentArtifactID, parentVersion, allProperties, cfg) - - if err == nil { - if parentPom != nil { - parsedPomFiles[pomCoordinates] = true - addMissingPropertiesFromProject(allProperties, parentPom) - - // recurse into another parent pom - if parentPom.Parent != nil { - getPropertiesFromParentPoms(ctx, allProperties, *parentPom.GroupID, *parentPom.ArtifactID, *parentPom.Version, - cfg, parsedPomFiles) - } - } else { - log.Errorf("got empty parent pom, error: %w") - } - } else { - log.Errorf("could not get parent pom: %w", err) +// directManagedDependencies returns all managed dependencies in a project, including all defined in profiles +func directManagedDependencies(pom *gopom.Project) []gopom.Dependency { + var dependencies []gopom.Dependency + if pom.DependencyManagement != nil { + dependencies = append(dependencies, deref(pom.DependencyManagement.Dependencies)...) } -} - -// Resolve references to properties (e.g. '${prop.name}') recursively by searching entries in 'allProperties'. -func resolveRecursiveByPropertyName(pomProperties map[string]string, propertyName string) string { - if strings.HasPrefix(propertyName, "${") { - name := getPropertyName(propertyName) - if value, ok := pomProperties[name]; ok { - if strings.HasPrefix(value, "${") { - return resolveRecursiveByPropertyName(pomProperties, value) - } - return value + for _, profile := range deref(pom.Profiles) { + if profile.DependencyManagement != nil { + dependencies = append(dependencies, deref(profile.DependencyManagement.Dependencies)...) } } - return propertyName + return dependencies } -// If 'value' is a property reference (e.g. '${prop.name}'), return the property name (e.g. prop.name). -// Otherwise return the given 'value' -func getPropertyName(value string) string { - propertyName := value - if strings.HasPrefix(propertyName, "${") { - propertyName = strings.TrimSpace(propertyName[2 : len(propertyName)-1]) // remove leading ${ and trailing } - } - return propertyName -} - -// Add all properties from the project 'pom' to the map 'allProperties' that are not already in the map. -func addMissingPropertiesFromProject(allProperties map[string]string, pom *gopom.Project) { - if pom != nil && pom.Properties != nil && pom.Properties.Entries != nil { - for name := range pom.Properties.Entries { - // Add property from pom that is not yet in allProperties map. - _, exists := allProperties[name] - if !exists { - currentValue := pom.Properties.Entries[name] - value := resolveProperty(*pom, ¤tValue, getPropertyName(currentValue)) - allProperties[name] = value - // log.Tracef("added property ['%s'='%s'] from pom [%s, %s, %s] to allProperties", name, value, - // *pom.GroupID, *pom.ArtifactID, *pom.Version) - } - } - // Try to resolve any added properties containing property references. - for name, value := range allProperties { - if strings.HasPrefix(value, "${") { - allProperties[name] = resolveRecursiveByPropertyName(allProperties, value) - } - } - } -} - -// Add all properties from map 'allProperties' to the project 'pom' that are not already defined in the pom. -// This increases the chance of the 'resolveProperty' function succeeding. -func addPropertiesToProject(pom *gopom.Project, allProperties map[string]string) { - if len(allProperties) > 0 { - if pom.Properties == nil { - var props gopom.Properties - props.Entries = make(map[string]string) - pom.Properties = &props - } - for name, value := range allProperties { - _, exists := pom.Properties.Entries[name] - if !exists { - pom.Properties.Entries[name] = value - // log.Tracef("added property ['%s'='%s'] to pom [%s, %s, %s]", name, value, *pom.GroupID, *pom.ArtifactID, *pom.Version) - } - } +// deref dereferences ptr if not nil, or returns the type default value if ptr is nil +func deref[T any](ptr *T) T { + if ptr == nil { + var t T + return t } + return *ptr } diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go new file mode 100644 index 00000000000..38f73b8e467 --- /dev/null +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -0,0 +1,188 @@ +package java + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/vifraa/gopom" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/cache" + "github.com/anchore/syft/internal/log" +) + +// mavenID is the unique identifier for a package in Maven +type mavenID struct { + GroupID string + ArtifactID string + Version string +} + +func (m mavenID) String() string { + return fmt.Sprintf("%s:%s:%s", m.GroupID, m.ArtifactID, m.Version) +} + +func newMavenID(groupID, artifactID, version *string) mavenID { + return mavenID{ + GroupID: deref(groupID), + ArtifactID: deref(artifactID), + Version: deref(version), + } +} + +// mavenResolver is a short-lived utility to resolve maven poms from multiple sources, including: +// the scanned filesystem, local maven cache directories, remote maven repositories, and the syft cache +type mavenResolver struct { + cfg ArchiveCatalogerConfig + // resolver file.Resolver + cache cache.Resolver[*gopom.Project] + resolved map[mavenID]*gopom.Project + remoteRequestTimeout time.Duration + checkedLocalRepo bool +} + +func newMavenResolver(cfg ArchiveCatalogerConfig) mavenResolver { + return mavenResolver{ + cfg: cfg, + cache: cache.GetResolver[*gopom.Project]("java/maven/pom", "v1"), + resolved: map[mavenID]*gopom.Project{}, + remoteRequestTimeout: time.Second * 10, + } +} + +// findPom gets a pom from cache, local repository, or downloads from a remote Maven repository depending on configuration +func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { + if groupID == "" || artifactID == "" || version == "" { + return nil, fmt.Errorf("invalid parent specification, require non-empty values for groupID :'%s', artifactID :'%s', version :'%s'", groupID, artifactID, version) + } + + id := newMavenID(&groupID, &artifactID, &version) + pom := r.resolved[id] + + if pom != nil { + return pom, nil + } + + var errs error + + // try to resolve first from local maven repo + if r.cfg.UseMavenLocalRepository { + pom, err := r.findPomInLocalRepository(groupID, artifactID, version) + if pom != nil { + r.resolved[id] = pom + return pom, nil + } + errs = errors.Join(errs, err) + } + + // resolve via network maven repository + if pom == nil && r.cfg.UseNetwork { + pom, err := r.findPomInRemoteRepo(ctx, groupID, artifactID, version) + if pom != nil { + r.resolved[id] = pom + return pom, nil + } + errs = errors.Join(errs, err) + } + + if errs != nil { + return nil, fmt.Errorf("unable to resolve pom %s %s %s: %w", groupID, artifactID, version, errs) + } + + return nil, nil +} + +// Try to get the Pom from the users local repository in the users home dir. +// Returns (nil, false) when file cannot be found or read for any reason. +func (r *mavenResolver) findPomInLocalRepository(groupID, artifactID, version string) (*gopom.Project, error) { + groupPath := filepath.Join(strings.Split(groupID, ".")...) + pomFilePath := filepath.Join(r.cfg.MavenLocalRepositoryDir, groupPath, artifactID, version, artifactID+"-"+version+".pom") + pomFile, err := os.Open(pomFilePath) + if err != nil { + if !r.checkedLocalRepo && errors.Is(err, os.ErrNotExist) { + r.checkedLocalRepo = true + // check if the directory exists at all, and if not just stop trying to resolve local maven files + fi, err := os.Stat(r.cfg.MavenLocalRepositoryDir) + if errors.Is(err, os.ErrNotExist) || !fi.IsDir() { + log.Debugf("local maven repository is not a readable directory, stopping local resolution: %v", r.cfg.MavenLocalRepositoryDir) + r.cfg.UseMavenLocalRepository = false + } + } + return nil, err + } + defer internal.CloseAndLogError(pomFile, pomFilePath) + + return decodePomXML(pomFile) +} + +// Download the pom file from a (remote) Maven repository over HTTP. +func (r *mavenResolver) findPomInRemoteRepo(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { + if groupID == "" || artifactID == "" || version == "" { + return nil, fmt.Errorf("missing/incomplete maven artifact coordinates -- groupId: '%s' artifactId: '%s', version: '%s'", groupID, artifactID, version) + } + + key := fmt.Sprintf("%s:%s:%s", groupID, artifactID, version) + + // Downloading snapshots requires additional steps to determine the latest snapshot version. + // See: https://maven.apache.org/ref/3-LATEST/maven-repository-metadata/ + if strings.HasSuffix(version, "-SNAPSHOT") { + return nil, fmt.Errorf("downloading snapshot artifacts is not supported, got: %s", key) + } + + return r.cache.Resolve(key, func() (*gopom.Project, error) { + requestURL, err := r.remotePomURL(groupID, artifactID, version) + if err != nil { + return nil, err + } + log.Debugf("fetching parent pom from maven repository, at url: %s", requestURL) + + req, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, fmt.Errorf("unable to create request for Maven central: %w", err) + } + + req = req.WithContext(ctx) + + client := http.Client{ + Timeout: r.remoteRequestTimeout, + } + + resp, err := client.Do(req) //nolint:bodyclose + if err != nil { + return nil, fmt.Errorf("unable to get pom from Maven repository %v: %w", requestURL, err) + } + defer internal.CloseAndLogError(resp.Body, requestURL) + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("pom not found in Maven repository at: %v", requestURL) + } + + pom, err := decodePomXML(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to parse pom from Maven repository url %v: %w", requestURL, err) + } + return pom, nil + }) +} + +func (r *mavenResolver) remotePomURL(groupID, artifactID, version string) (requestURL string, err error) { + return remotePomURL(r.cfg.MavenBaseURL, groupID, artifactID, version) +} + +func (r *mavenResolver) findParent(ctx context.Context, pom *gopom.Project) (*gopom.Project, error) { + if pom == nil || pom.Parent == nil { + return nil, nil + } + parent := pom.Parent + pomWithoutParent := *pom + pomWithoutParent.Parent = nil + groupID := r.getPropertyValue(ctx, &pomWithoutParent, parent.GroupID) + artifactID := r.getPropertyValue(ctx, &pomWithoutParent, parent.ArtifactID) + version := r.getPropertyValue(ctx, &pomWithoutParent, parent.Version) + return r.findPom(ctx, groupID, artifactID, version) +} diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go new file mode 100644 index 00000000000..39427027516 --- /dev/null +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -0,0 +1,124 @@ +package java + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/bmatcuk/doublestar/v4" + "github.com/stretchr/testify/require" +) + +func Test_mavenResolverLocal(t *testing.T) { + dir, err := filepath.Abs("test-fixtures/pom/maven-repo") + require.NoError(t, err) + + tests := []struct { + groupID string + artifactID string + version string + expression string + expected string + wantErr require.ErrorAssertionFunc + }{ + { + groupID: "my.org", + artifactID: "child-one", + version: "1.3.6", + expression: "${project.one}", + expected: "1", + }, + } + + for _, test := range tests { + t.Run(test.artifactID, func(t *testing.T) { + ctx := context.Background() + r := newMavenResolver(ArchiveCatalogerConfig{ + UseNetwork: false, + UseMavenLocalRepository: true, + MavenLocalRepositoryDir: dir, + MaxParentRecursiveDepth: 5, + }) + pom, err := r.findPom(ctx, test.groupID, test.artifactID, test.version) + if test.wantErr != nil { + test.wantErr(t, err) + } else { + require.NoError(t, err) + } + got := r.getPropertyValue(context.Background(), pom, &test.expression) + require.Equal(t, test.expected, got) + }) + } +} + +func Test_mavenResolverRemote(t *testing.T) { + url := testRepo(t, "test-fixtures/pom/maven-repo") + + tests := []struct { + groupID string + artifactID string + version string + expression string + expected string + wantErr require.ErrorAssertionFunc + }{ + { + groupID: "my.org", + artifactID: "child-one", + version: "1.3.6", + expression: "${project.one}", + expected: "1", + }, + } + + for _, test := range tests { + t.Run(test.artifactID, func(t *testing.T) { + ctx := context.Background() + r := newMavenResolver(ArchiveCatalogerConfig{ + UseNetwork: true, + UseMavenLocalRepository: false, + MavenBaseURL: url, + MaxParentRecursiveDepth: 5, + }) + pom, err := r.findPom(ctx, test.groupID, test.artifactID, test.version) + if test.wantErr != nil { + test.wantErr(t, err) + } else { + require.NoError(t, err) + } + got := r.getPropertyValue(context.Background(), pom, &test.expression) + require.Equal(t, test.expected, got) + }) + } +} + +// testRepo starts a remote maven repo serving all the pom files found in the given directory +func testRepo(t *testing.T, dir string) (url string) { + // mux is the HTTP request multiplexer used with the test server. + mux := http.NewServeMux() + + // We want to ensure that tests catch mistakes where the endpoint URL is + // specified as absolute rather than relative. It only makes a difference + // when there's a non-empty base URL path. So, use that. See issue #752. + apiHandler := http.NewServeMux() + apiHandler.Handle("/", mux) + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + t.Cleanup(server.Close) + + matches, err := doublestar.Glob(os.DirFS(dir), filepath.Join("**", "*.pom")) + require.NoError(t, err) + + for _, match := range matches { + fullPath, err := filepath.Abs(filepath.Join(dir, match)) + require.NoError(t, err) + match = "/" + filepath.ToSlash(match) + mux.HandleFunc(match, generateMockMavenHandler(fullPath)) + } + + return server.URL +} diff --git a/syft/pkg/cataloger/java/maven_utils.go b/syft/pkg/cataloger/java/maven_utils.go new file mode 100644 index 00000000000..ee4e93f8f6e --- /dev/null +++ b/syft/pkg/cataloger/java/maven_utils.go @@ -0,0 +1,47 @@ +package java + +import ( + "encoding/xml" + "io" + "os" + "path/filepath" + + "github.com/mitchellh/go-homedir" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" +) + +// defaultMavenLocalRepoDir gets default location of the Maven local repository, generally at /.m2/repository +func defaultMavenLocalRepoDir() string { + homeDir, err := homedir.Dir() + if err != nil { + return "" + } + + mavenHome := filepath.Join(homeDir, ".m2") + + settingsXML := filepath.Join(mavenHome, "settings.xml") + settings, err := os.Open(settingsXML) + if err == nil && settings != nil { + defer internal.CloseAndLogError(settings, settingsXML) + localRepository := getSettingsXMLLocalRepository(settings) + if localRepository != "" { + return localRepository + } + } + return filepath.Join(mavenHome, "repository") +} + +// getSettingsXMLLocalRepository reads the provided settings.xml and parses the localRepository, if present +func getSettingsXMLLocalRepository(settingsXML io.Reader) string { + type settings struct { + LocalRepository string `xml:"localRepository"` + } + s := settings{} + err := xml.NewDecoder(settingsXML).Decode(&s) + if err != nil { + log.Debugf("unable to read maven settings.xml: %v", err) + } + return s.LocalRepository +} diff --git a/syft/pkg/cataloger/java/maven_utils_test.go b/syft/pkg/cataloger/java/maven_utils_test.go new file mode 100644 index 00000000000..47028d9f147 --- /dev/null +++ b/syft/pkg/cataloger/java/maven_utils_test.go @@ -0,0 +1,77 @@ +package java + +import ( + "os" + "path/filepath" + "testing" + + "github.com/mitchellh/go-homedir" + "github.com/stretchr/testify/require" +) + +func Test_defaultMavenLocalRepoDir(t *testing.T) { + home, err := homedir.Dir() + require.NoError(t, err) + + fixtures, err := filepath.Abs("test-fixtures") + require.NoError(t, err) + + tests := []struct { + name string + home string + expected string + }{ + { + name: "default", + expected: filepath.Join(home, ".m2", "repository"), + home: "", + }, + { + name: "alternate dir", + expected: "/some/other/repo", + home: "test-fixtures/local-repository-settings", + }, + { + name: "explicit home", + expected: filepath.Join(fixtures, ".m2", "repository"), + home: "test-fixtures", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + homedir.Reset() + defer homedir.Reset() + if test.home != "" { + home, err := filepath.Abs(test.home) + require.NoError(t, err) + t.Setenv("HOME", home) + } + got := defaultMavenLocalRepoDir() + require.Equal(t, test.expected, got) + }) + } +} + +func Test_getSettingsXmlLocalRepository(t *testing.T) { + tests := []struct { + file string + expected string + }{ + { + expected: "/some/other/repo", + file: "test-fixtures/local-repository-settings/.m2/settings.xml", + }, + { + expected: "", + file: "invalid", + }, + } + for _, test := range tests { + t.Run(test.expected, func(t *testing.T) { + f, _ := os.Open(test.file) + defer f.Close() + got := getSettingsXMLLocalRepository(f) + require.Equal(t, test.expected, got) + }) + } +} diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index fb959504769..d3ce52011ab 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "encoding/xml" + "errors" "fmt" "io" "reflect" "regexp" + "slices" "strings" "github.com/saintfish/chardet" @@ -23,42 +25,34 @@ import ( const pomXMLGlob = "*pom.xml" -var propertyMatcher = regexp.MustCompile("[$][{][^}]+[}]") +var expressionMatcher = regexp.MustCompile("[$][{][^}]+[}]") -func (gap genericArchiveParserAdapter) parserPomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) - if err != nil { + if err != nil || pom == nil { return nil, nil, err } - var pkgs []pkg.Package + r := newMavenResolver(gap.cfg) - // Add all properties defined in parent poms to this project for resolving properties later on. - if pom.Parent != nil { - var allProperties = make(map[string]string) - getPropertiesFromParentPoms( - ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, gap.cfg, nil) - addPropertiesToProject(&pom, allProperties) - } + var pkgs []pkg.Package - for _, dep := range *getPomDependencies(&pom) { - log.Debugf("add dependency to SBOM : [%s, %s, %s]", *dep.GroupID, *dep.ArtifactID, safeString(dep.Version)) - p := newPackageFromPom( + for _, dep := range directDependencies(pom) { + id := newMavenID(dep.GroupID, dep.ArtifactID, dep.Version) + log.Debugf("add dependency to SBOM : [%v]", id) + p, err := r.newPackageFromDependency( ctx, pom, dep, - gap.cfg, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - if p.Name == "" { - continue + if err != nil { + log.Debugf("error adding dependency %v: %v", id, err) } - - pkgs = append(pkgs, p) - - if len(p.Version) == 0 || strings.HasPrefix(p.Version, "${") { - log.Infof("found artifact without version: %s:%s, version: %q", *dep.GroupID, *dep.ArtifactID, p.Version) + if p == nil { + continue } + pkgs = append(pkgs, *p) } return pkgs, nil, nil @@ -70,25 +64,19 @@ func parsePomXMLProject(ctx context.Context, path string, reader io.Reader, loca return nil, err } - // Add all properties defined in parent poms to this project for resolving properties later on. - if pom.Parent != nil { - var allProperties = make(map[string]string) - getPropertiesFromParentPoms( - ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg, nil) - addPropertiesToProject(&pom, allProperties) - } + resolver := newMavenResolver(cfg) - return newPomProject(path, pom, location), nil + return resolver.newPomProject(ctx, path, pom, location), nil } -func newPomProject(path string, p gopom.Project, location file.Location) *parsedPomProject { - artifactID := safeString(p.ArtifactID) - name := safeString(p.Name) - projectURL := safeString(p.URL) +func (r *mavenResolver) newPomProject(ctx context.Context, path string, pom *gopom.Project, location file.Location) *parsedPomProject { + artifactID := deref(pom.ArtifactID) + name := deref(pom.Name) + projectURL := deref(pom.URL) var licenses []pkg.License - if p.Licenses != nil { - for _, license := range *p.Licenses { + if pom.Licenses != nil { + for _, license := range *pom.Licenses { var licenseName, licenseURL string if license.Name != nil { licenseName = *license.Name @@ -109,76 +97,57 @@ func newPomProject(path string, p gopom.Project, location file.Location) *parsed return &parsedPomProject{ JavaPomProject: &pkg.JavaPomProject{ Path: path, - Parent: pomParent(p, p.Parent), - GroupID: resolveProperty(p, p.GroupID, "groupId"), + Parent: r.pomParent(ctx, pom), + GroupID: r.getPropertyValue(ctx, pom, pom.GroupID), ArtifactID: artifactID, - Version: resolveProperty(p, p.Version, "version"), + Version: r.getPropertyValue(ctx, pom, pom.Version), Name: name, - Description: cleanDescription(p.Description), + Description: cleanDescription(pom.Description), URL: projectURL, }, Licenses: licenses, } } -func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Dependency, cfg ArchiveCatalogerConfig, locations ...file.Location) pkg.Package { - groupID := resolveProperty(pom, dep.GroupID, "groupId") - artifactID := resolveProperty(pom, dep.ArtifactID, "artifactId") +func (r *mavenResolver) newPackageFromDependency(ctx context.Context, pom *gopom.Project, dep gopom.Dependency, locations ...file.Location) (*pkg.Package, error) { + groupID := r.getPropertyValue(ctx, pom, dep.GroupID) + artifactID := r.getPropertyValue(ctx, pom, dep.ArtifactID) + version := r.getPropertyValue(ctx, pom, dep.Version) + + var err error + if version == "" { + version, err = r.findInheritedVersion(ctx, pom, pom, groupID, artifactID) + } m := pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: groupID, ArtifactID: artifactID, - Scope: resolveProperty(pom, dep.Scope, "scope"), + Scope: r.getPropertyValue(ctx, pom, dep.Scope), }, } - name := safeString(dep.ArtifactID) - version := resolveProperty(pom, dep.Version, "version") - var allProperties = make(map[string]string) - addMissingPropertiesFromProject(allProperties, &pom) - licenses := make([]pkg.License, 0) if version == "" { - // If we have no version then let's try to get it from a parent pom DependencyManagement section - version = recursivelyFindVersionFromManagedOrInherited(ctx, *dep.GroupID, *dep.ArtifactID, &pom, cfg, allProperties, nil) - version = resolveProperty(pom, &version, "version") - } else if strings.HasPrefix(version, "${") { - // If we are missing the property for this version, search the pom hierarchy for it. - if pom.Parent != nil { - getPropertiesFromParentPoms(ctx, allProperties, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, - cfg, nil) - addPropertiesToProject(&pom, allProperties) + dependencyPom, depErr := r.findPom(ctx, groupID, artifactID, version) + if depErr != nil { + log.Debugf("error getting licenses for %s: %v", mavenID{groupID, artifactID, version}, err) + err = errors.Join(err, depErr) } - version = resolveProperty(pom, &version, getPropertyName(version)) - } - if isPropertyResolved(version) { - parentLicenses, _ := recursivelyFindLicensesFromParentPom( - ctx, - m.PomProperties.GroupID, - m.PomProperties.ArtifactID, - version, - cfg) - - if len(parentLicenses) > 0 { + if dependencyPom != nil { + parentLicenses, _ := r.findLicenses(ctx, dependencyPom) for _, licenseName := range parentLicenses { licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) } } - } else { - log.Warnf("could not determine version for package: [%s, %s]", groupID, artifactID) } - if strings.HasPrefix(version, "${") { - log.Infof("got unresolved version '%s' for artifact: %s", version, name) - } - - p := pkg.Package{ - Name: name, + p := &pkg.Package{ + Name: artifactID, Version: version, Locations: file.NewLocationSet(locations...), Licenses: pkg.NewLicenseSet(licenses...), - PURL: packageURL(name, version, m), + PURL: packageURL(artifactID, version, m), Language: pkg.Java, Type: pkg.JavaPkg, // TODO: should we differentiate between packages from jar/war/zip versus packages from a pom.xml that were not installed yet? Metadata: m, @@ -186,21 +155,22 @@ func newPackageFromPom(ctx context.Context, pom gopom.Project, dep gopom.Depende p.SetID() - return p + return p, err } -func decodePomXML(content io.Reader) (project gopom.Project, err error) { +func decodePomXML(content io.Reader) (project *gopom.Project, err error) { inputReader, err := getUtf8Reader(content) if err != nil { - return project, fmt.Errorf("unable to read pom.xml: %w", err) + return nil, fmt.Errorf("unable to read pom.xml: %w", err) } decoder := xml.NewDecoder(inputReader) // when an xml file has a character set declaration (e.g. '') read that and use the correct decoder decoder.CharsetReader = charset.NewReaderLabel - if err := decoder.Decode(&project); err != nil { - return project, fmt.Errorf("unable to unmarshal pom.xml: %w", err) + project = &gopom.Project{} + if err := decoder.Decode(project); err != nil { + return nil, fmt.Errorf("unable to unmarshal pom.xml: %w", err) } // For modules groupID and version are almost always inherited from parent pom @@ -211,8 +181,6 @@ func decodePomXML(content io.Reader) (project gopom.Project, err error) { project.Version = project.Parent.Version } - // Store in cache - parsedPomFilesCache[mavenCoordinate{*project.GroupID, *project.ArtifactID, *project.Version}] = &project return project, nil } @@ -243,22 +211,23 @@ func getUtf8Reader(content io.Reader) (io.Reader, error) { return inputReader, nil } -func pomParent(pom gopom.Project, parent *gopom.Parent) (result *pkg.JavaPomParent) { - if parent == nil { +func (r *mavenResolver) pomParent(ctx context.Context, pom *gopom.Project) *pkg.JavaPomParent { + if pom == nil || pom.Parent == nil { return nil } - artifactID := safeString(parent.ArtifactID) - result = &pkg.JavaPomParent{ - GroupID: resolveProperty(pom, parent.GroupID, "groupId"), - ArtifactID: artifactID, - Version: resolveProperty(pom, parent.Version, "version"), + groupID := deref(pom.Parent.GroupID) + artifactID := deref(pom.Parent.ArtifactID) + version := deref(pom.Parent.Version) + if groupID == "" && artifactID == "" && version == "" { + return nil } - if result.GroupID == "" && result.ArtifactID == "" && result.Version == "" { - return nil + return &pkg.JavaPomParent{ + GroupID: r.getPropertyValue(ctx, pom, pom.Parent.GroupID), + ArtifactID: artifactID, + Version: r.getPropertyValue(ctx, pom, pom.Parent.Version), } - return result } func cleanDescription(original *string) (cleaned string) { @@ -276,107 +245,137 @@ func cleanDescription(original *string) (cleaned string) { return strings.TrimSpace(cleaned) } -// resolveProperty emulates some maven property resolution logic by looking in the project's variables +// getPropertyValue gets property values by emulating maven property resolution logic, looking in the project's variables // as well as supporting the project expressions like ${project.parent.groupId}. -// If no match is found, the entire expression including ${} is returned +// Properties which are not resolved result in empty string "" // //nolint:gocognit -func resolveProperty(pom gopom.Project, propertyValue *string, propertyName string) string { - propertyCase := safeString(propertyValue) - if !strings.Contains(propertyCase, "${") { - // nothing to resolve - // log.Tracef("resolving property: value [%s] contains no variable", propertyName) - return propertyCase - } - - log.WithFields("existingPropertyValue", propertyCase, "propertyName", propertyName).Trace("resolving property") - seenBeforePropertyNames := map[string]struct{}{ - propertyName: {}, +func (r *mavenResolver) getPropertyValue(ctx context.Context, pom *gopom.Project, propertyValue *string) string { + if propertyValue == nil { + return "" } - result := recursiveResolveProperty(pom, propertyCase, seenBeforePropertyNames) - if propertyMatcher.MatchString(result) { - return "" // dereferencing variable failed; fall back to empty string + resolved, err := r.resolveExpression(ctx, pom, *propertyValue, nil) + if err != nil { + log.Debugf("error resolving maven property: %s: %v", *propertyValue, err) + return "" } - return result + return resolved } +// resolveExpression resolves an expression, which may be a plain string or a string with ${ property.references } +// //nolint:gocognit -func recursiveResolveProperty(pom gopom.Project, propertyCase string, seenPropertyNames map[string]struct{}) string { - return propertyMatcher.ReplaceAllStringFunc(propertyCase, func(match string) string { - propertyName := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } - if _, seen := seenPropertyNames[propertyName]; seen { - return propertyCase +func (r *mavenResolver) resolveExpression(ctx context.Context, pom *gopom.Project, expression string, resolving []string) (string, error) { + var err error + return expressionMatcher.ReplaceAllStringFunc(expression, func(match string) string { + propertyExpression := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } + resolved, e := r.resolveProperty(ctx, pom, propertyExpression, resolving) + if e != nil { + err = errors.Join(err, e) + return "" + } + return resolved + }), err +} + +// resolveProperty resolves properties recursively from the root project +// +//nolint:gocognit +func (r *mavenResolver) resolveProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { + // prevent cycles + if slices.Contains(resolving, propertyExpression) { + return "", fmt.Errorf("cycle detected resolving: %s", propertyExpression) + } + resolving = append(resolving, propertyExpression) + + value, err := r.resolveProjectProperty(ctx, pom, propertyExpression, resolving) + if err != nil { + return value, err + } + if value != "" { + return value, nil + } + + current := pom + for current != nil { + if current.Properties != nil && current.Properties.Entries != nil { + if value, ok := current.Properties.Entries[propertyExpression]; ok { + return r.resolveExpression(ctx, pom, value, resolving) // property values can contain expressions + } } - entries := pomProperties(pom) - if value, ok := entries[propertyName]; ok { - seenPropertyNames[propertyName] = struct{}{} - value = recursiveResolveProperty(pom, value, seenPropertyNames) // recursively resolve in case a variable points to a variable. - //if isPropertyResolved(value) { - log.WithFields("propertyValue", value, "propertyName", match).Trace("resolved property") - return value - //} + current, err = r.findParent(ctx, current) + if err != nil { + return "", err } + } - // if we don't find anything directly in the pom properties, - // see if we have a project.x expression and process this based - // on the xml tags in gopom - parts := strings.Split(propertyName, ".") - numParts := len(parts) - if numParts > 1 && strings.TrimSpace(parts[0]) == "project" { - pomValue := reflect.ValueOf(pom) - pomValueType := pomValue.Type() - for partNum := 1; partNum < numParts; partNum++ { - if pomValueType.Kind() != reflect.Struct { - break + return "", fmt.Errorf("unable to resolve property: %s", propertyExpression) +} + +// resolveProjectProperty resolves properties on the project +// +//nolint:gocognit +func (r *mavenResolver) resolveProjectProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { + // see if we have a project.x expression and process this based + // on the xml tags in gopom + parts := strings.Split(propertyExpression, ".") + numParts := len(parts) + if numParts > 1 && strings.TrimSpace(parts[0]) == "project" { + pomValue := reflect.ValueOf(pom).Elem() + pomValueType := pomValue.Type() + for partNum := 1; partNum < numParts; partNum++ { + if pomValueType.Kind() != reflect.Struct { + break + } + + part := parts[partNum] + for fieldNum := 0; fieldNum < pomValueType.NumField(); fieldNum++ { + f := pomValueType.Field(fieldNum) + tag := f.Tag.Get("xml") + tag = strings.Split(tag, ",")[0] + // a segment of the property name matches the xml tag for the field, + // so we need to recurse down the nested structs or return a match + // if we're done. + if part != tag { + continue } - part := parts[partNum] - for fieldNum := 0; fieldNum < pomValueType.NumField(); fieldNum++ { - f := pomValueType.Field(fieldNum) - tag := f.Tag.Get("xml") - tag = strings.Split(tag, ",")[0] - // a segment of the property name matches the xml tag for the field, - // so we need to recurse down the nested structs or return a match - // if we're done. - if part == tag { - pomValue = pomValue.Field(fieldNum) + + pomValue = pomValue.Field(fieldNum) + pomValueType = pomValue.Type() + if pomValueType.Kind() == reflect.Ptr { + // we were recursing down the nested structs, but one of the steps + // we need to take is a nil pointer, so give up + if pomValue.IsNil() { + return "", fmt.Errorf("property undefined: %s", propertyExpression) + } + pomValue = pomValue.Elem() + if !pomValue.IsZero() { + // we found a non-zero value whose tag matches this part of the property name pomValueType = pomValue.Type() - if pomValueType.Kind() == reflect.Ptr { - // we were recursing down the nested structs, but one of the steps - // we need to take is a nil pointer, so give up and return the original match - if pomValue.IsNil() { - return match - } - pomValue = pomValue.Elem() - if !pomValue.IsZero() { - // we found a non-zero value whose tag matches this part of the property name - pomValueType = pomValue.Type() - } - } - // If this was the last part of the property name, return the value - if partNum == numParts-1 { - value := fmt.Sprintf("%v", pomValue.Interface()) - log.WithFields("existingPropertyValue", value, "propertyName", propertyName).Trace("resolved property") - return value - } - break } } + // If this was the last part of the property name, return the value + if partNum == numParts-1 { + value := fmt.Sprintf("%v", pomValue.Interface()) + return r.resolveExpression(ctx, pom, value, resolving) + } + break } } - return match - }) -} - -func pomProperties(p gopom.Project) map[string]string { - if p.Properties != nil { - return p.Properties.Entries } - return map[string]string{} + return "", nil } -func safeString(s *string) string { - if s == nil { - return "" - } - return *s -} +// func pomProperties(p gopom.Project) map[string]string { +// if p.Properties != nil { +// return p.Properties.Entries +// } +// return map[string]string{} +//} + +// func deref(s *string) string { +// if s == nil { +// return "" +// } +// return *s +//} diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 31a9e657ef3..efc28c5c09b 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -19,8 +19,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) -func Test_parserPomXML(t *testing.T) { - resetRepoUtils() +func Test_parsePomXML(t *testing.T) { tests := []struct { input string expected []pkg.Package @@ -72,7 +71,7 @@ func Test_parserPomXML(t *testing.T) { }, }) - pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) + pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) }) } } @@ -156,21 +155,20 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { }, UseMavenLocalRepository: false, }) - pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) + pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) }) } } func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { - resetRepoUtils() // Using the local repository, the version of junit-jupiter will be resolved expectedPackages := getCommonsTextExpectedPackages() for i := 0; i < len(expectedPackages); i++ { if expectedPackages[i].Name == "junit-jupiter" { expPkg := &expectedPackages[i] - expPkg.Version = "5.9.0" - expPkg.PURL = "pkg:maven/org.junit.jupiter/junit-jupiter@5.9.0" + expPkg.Version = "5.9.1" + expPkg.PURL = "pkg:maven/org.junit.jupiter/junit-jupiter@5.9.1" expPkg.Metadata = pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.junit.jupiter", @@ -204,24 +202,24 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { }, UseMavenLocalRepository: true, MavenLocalRepositoryDir: "test-fixtures/pom/maven-repo", + MaxParentRecursiveDepth: 5, }) - pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) + pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) }) } } func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { - resetRepoUtils() - mux, url, teardown := setup() - defer teardown() + url := testRepo(t, "test-fixtures/pom/maven-repo") + // Using the local repository, the version of junit-jupiter will be resolved expectedPackages := getCommonsTextExpectedPackages() for i := 0; i < len(expectedPackages); i++ { if expectedPackages[i].Name == "junit-jupiter" { expPkg := &expectedPackages[i] - expPkg.Version = "5.9.0" - expPkg.PURL = "pkg:maven/org.junit.jupiter/junit-jupiter@5.9.0" + expPkg.Version = "5.9.1" + expPkg.PURL = "pkg:maven/org.junit.jupiter/junit-jupiter@5.9.1" expPkg.Metadata = pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.junit.jupiter", @@ -233,33 +231,17 @@ func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { } tests := []struct { - input string - expected []pkg.Package - requestHandlers []handlerPath + input string + expected []pkg.Package }{ { input: "test-fixtures/pom/commons-text.pom.xml", expected: expectedPackages, - requestHandlers: []handlerPath{ - { - path: "/org/apache/commons/commons-parent/54/commons-parent-54.pom", - handler: generateMockMavenHandler("test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom"), - }, - { - path: "/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom", - handler: generateMockMavenHandler("test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom"), - }, - }, }, } for _, test := range tests { t.Run(test.input, func(t *testing.T) { - // configure maven central requests - for _, hdlr := range test.requestHandlers { - mux.HandleFunc(hdlr.path, hdlr.handler) - } - for i := range test.expected { test.expected[i].Locations.Add(file.NewLocation(test.input)) } @@ -272,14 +254,14 @@ func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { UseNetwork: true, MavenBaseURL: url, UseMavenLocalRepository: false, + MaxParentRecursiveDepth: 5, }) - pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) + pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) }) } } func Test_parsePomXMLProject(t *testing.T) { - resetRepoUtils() // TODO: ideally we would have the path to the contained pom.xml, not the jar jarLocation := file.NewLocation("path/to/archive.jar") tests := []struct { @@ -414,7 +396,8 @@ func Test_pomParent(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, pomParent(gopom.Project{}, test.input)) + r := newMavenResolver(DefaultArchiveCatalogerConfig()) + assert.Equal(t, test.expected, r.pomParent(context.Background(), &gopom.Project{Parent: test.input})) }) } } @@ -564,7 +547,7 @@ func Test_resolveProperty(t *testing.T) { expected: "", }, { - name: "resolution halts even if cyclic involving parent", + name: "resolution halts even if cyclic involving parent", property: "${cyclic.version}", pom: gopom.Project{ Parent: &gopom.Parent{ @@ -584,7 +567,8 @@ func Test_resolveProperty(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - resolved := resolveProperty(test.pom, stringPointer(test.property), test.name) + r := newMavenResolver(DefaultArchiveCatalogerConfig()) + resolved := r.getPropertyValue(context.Background(), &test.pom, stringPointer(test.property)) assert.Equal(t, test.expected, resolved) }) } @@ -762,10 +746,3 @@ func getCommonsTextExpectedPackages() []pkg.Package { }, } } - -// Reset caches in maven_repo_utils for independent unit tests. -func resetRepoUtils() { - parsedPomFilesCache = make(map[mavenCoordinate]*gopom.Project) - checkedForMavenLocalRepo = false - mavenLocalRepoDir = "" -} diff --git a/syft/pkg/cataloger/java/test-fixtures/local-repository-settings/.m2/settings.xml b/syft/pkg/cataloger/java/test-fixtures/local-repository-settings/.m2/settings.xml new file mode 100644 index 00000000000..f8b9552a4b7 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/local-repository-settings/.m2/settings.xml @@ -0,0 +1,4 @@ + + /some/other/repo + \ No newline at end of file diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom new file mode 100644 index 00000000000..2d42648e51b --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom @@ -0,0 +1,42 @@ + + + 4.0.0 + + + + + my.org + parent-one + 3.11.0 + ../../parent-1/pom.xml + + + child-one + + ${project.one}.3.6 + jar + + + 3.12.0 + 4.2 + 4.12 + + + + + org.apache.commons + commons-lang3 + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-two/2.1.90/child-two-2.1.90.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-two/2.1.90/child-two-2.1.90.pom new file mode 100644 index 00000000000..62a3592d181 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-two/2.1.90/child-two-2.1.90.pom @@ -0,0 +1,53 @@ + + + 4.0.0 + + + + + my.org + parent-one + 3.11.0 + + + ${project.parent.groupId} + child-two + 2.1.90 + jar + + + 4.2 + 4.12 + my.other.org + + + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-math${project.parent.version} + 3.5 + + + org.apache.commons + commons-exec + 1.${commons-exec_subversion} + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom new file mode 100644 index 00000000000..f4be4a02c43 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom @@ -0,0 +1,52 @@ + + + 4.0.0 + + my.org + parent-two + 13.7.8 + ../parent-2 + + + my.org + parent-one + 3.11.0 + pom + + + + 3.1${project.parent.version}.0 + 4.3 + + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-collections4 + ${commons.collections4.version} + + + junit + junit + ${commons.junit.version} + test + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom new file mode 100644 index 00000000000..fd7b3ba6c3e --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom @@ -0,0 +1,60 @@ + + + 4.0.0 + + my.org + parent-2 + 13 + pom + + + 3.14.0 + 4.4 + 1.12.0 + 4.13.2 + 3 + 1 + + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + org.apache.commons + commons-text + ${commons.text.version} + + + junit + junit + ${commons.junit.version} + test + + + + + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-collections4 + ${commons.collections4.version} + + + junit + junit + ${commons.junit.version} + test + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/maven-xml-responses/parent-7.11.2.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/net/shibboleth/parent/7.11.2/parent-7.11.2.pom similarity index 100% rename from syft/pkg/cataloger/java/test-fixtures/maven-xml-responses/parent-7.11.2.pom rename to syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/net/shibboleth/parent/7.11.2/parent-7.11.2.pom diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom new file mode 100644 index 00000000000..57e6b86cbb1 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom @@ -0,0 +1,159 @@ + + + + + + + + 4.0.0 + org.junit + junit-bom + 5.9.1 + pom + JUnit 5 (Bill of Materials) + This Bill of Materials POM can be used to ease dependency management when referencing multiple JUnit artifacts using Gradle or Maven. + https://junit.org/junit5/ + + + Eclipse Public License v2.0 + https://www.eclipse.org/legal/epl-v20.html + + + + + bechte + Stefan Bechtold + stefan.bechtold@me.com + + + jlink + Johannes Link + business@johanneslink.net + + + marcphilipp + Marc Philipp + mail@marcphilipp.de + + + mmerdes + Matthias Merdes + matthias.merdes@heidelpay.com + + + sbrannen + Sam Brannen + sam@sambrannen.com + + + sormuras + Christian Stein + sormuras@gmail.com + + + juliette-derancourt + Juliette de Rancourt + derancourt.juliette@gmail.com + + + + scm:git:git://github.com/junit-team/junit5.git + scm:git:git://github.com/junit-team/junit5.git + https://github.com/junit-team/junit5 + + + + + org.junit.jupiter + junit-jupiter + 5.9.1 + + + org.junit.jupiter + junit-jupiter-api + 5.9.1 + + + org.junit.jupiter + junit-jupiter-engine + 5.9.1 + + + org.junit.jupiter + junit-jupiter-migrationsupport + 5.9.1 + + + org.junit.jupiter + junit-jupiter-params + 5.9.1 + + + org.junit.platform + junit-platform-commons + 1.9.1 + + + org.junit.platform + junit-platform-console + 1.9.1 + + + org.junit.platform + junit-platform-engine + 1.9.1 + + + org.junit.platform + junit-platform-jfr + 1.9.1 + + + org.junit.platform + junit-platform-launcher + 1.9.1 + + + org.junit.platform + junit-platform-reporting + 1.9.1 + + + org.junit.platform + junit-platform-runner + 1.9.1 + + + org.junit.platform + junit-platform-suite + 1.9.1 + + + org.junit.platform + junit-platform-suite-api + 1.9.1 + + + org.junit.platform + junit-platform-suite-commons + 1.9.1 + + + org.junit.platform + junit-platform-suite-engine + 1.9.1 + + + org.junit.platform + junit-platform-testkit + 1.9.1 + + + org.junit.vintage + junit-vintage-engine + 5.9.1 + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/maven-xml-responses/opensaml-parent-3.4.6.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom similarity index 100% rename from syft/pkg/cataloger/java/test-fixtures/maven-xml-responses/opensaml-parent-3.4.6.pom rename to syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom From 2922853b0a63061c364c55986c60d0eb9b23a25a Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 12:17:42 -0400 Subject: [PATCH 24/42] chore: refactor maven_repo_utils Signed-off-by: Keith Zantow --- .../pkg/cataloger/java/archive_parser_test.go | 26 --- syft/pkg/cataloger/java/maven_repo_utils.go | 194 ------------------ syft/pkg/cataloger/java/maven_resolver.go | 161 +++++++++++++++ syft/pkg/cataloger/java/maven_utils.go | 27 +++ syft/pkg/cataloger/java/maven_utils_test.go | 26 +++ 5 files changed, 214 insertions(+), 220 deletions(-) delete mode 100644 syft/pkg/cataloger/java/maven_repo_utils.go diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 5c1870c3cdc..a5cadd16d1a 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -122,32 +122,6 @@ func TestSearchMavenForLicenses(t *testing.T) { } } -func Test_remotePomURL(t *testing.T) { - tests := []struct { - name string - groupID string - artifactID string - version string - expected string - }{ - { - name: "formatMavenURL correctly assembles the pom URL", - groupID: "org.springframework.boot", - artifactID: "spring-boot-starter-test", - version: "3.1.5", - expected: "https://repo1.maven.org/maven2/org/springframework/boot/spring-boot-starter-test/3.1.5/spring-boot-starter-test-3.1.5.pom", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - requestURL, err := remotePomURL(mavenBaseURL, tc.groupID, tc.artifactID, tc.version) - assert.NoError(t, err, "expected no err; got %w", err) - assert.Equal(t, tc.expected, requestURL) - }) - } -} - func TestParseJar(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go deleted file mode 100644 index 2ae86ef9018..00000000000 --- a/syft/pkg/cataloger/java/maven_repo_utils.go +++ /dev/null @@ -1,194 +0,0 @@ -package java - -import ( - "context" - "fmt" - "net/url" - "slices" - "strings" - - "github.com/vifraa/gopom" -) - -func remotePomURL(repoURL, groupID, artifactID, version string) (requestURL string, err error) { - // groupID needs to go from maven.org -> maven/org - urlPath := strings.Split(groupID, ".") - artifactPom := fmt.Sprintf("%s-%s.pom", artifactID, version) - urlPath = append(urlPath, artifactID, version, artifactPom) - - // ex: https://repo1.maven.org/maven2/groupID/artifactID/artifactPom - requestURL, err = url.JoinPath(repoURL, urlPath...) - if err != nil { - return requestURL, fmt.Errorf("could not construct maven url: %w", err) - } - return requestURL, err -} - -// Try to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies -// -//nolint:gocognit -func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Project, pom *gopom.Project, groupID, artifactID string, resolving ...mavenID) (string, error) { - id := newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) - if len(resolving) >= r.cfg.MaxParentRecursiveDepth { - return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) - } - if slices.Contains(resolving, id) { - return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) - } - resolving = append(resolving, id) - - var err error - var version string - - // check for entries in dependencyManagement first - for _, dep := range directManagedDependencies(pom) { - depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) - depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) - if depGroupID == groupID && depArtifactID == artifactID { - version = r.getPropertyValue(ctx, root, dep.Version) - if version != "" { - return version, nil - } - } - - // imported pom files should be treated just like parent poms, they are used to define versions of dependencies - if deref(dep.Type) == "pom" && deref(dep.Scope) == "import" { - depVersion := r.getPropertyValue(ctx, root, dep.Version) - - depPom, err := r.findPom(ctx, depGroupID, depArtifactID, depVersion) - if err != nil { - return "", err - } - version, err = r.findInheritedVersion(ctx, root, depPom, groupID, artifactID, resolving...) - if err != nil { - return "", err - } - if version != "" { - return version, nil - } - } - } - - // recursively check parents - parent, err := r.findParent(ctx, pom) - if err != nil { - return "", err - } - if parent != nil { - version, err = r.findInheritedVersion(ctx, root, parent, groupID, artifactID, resolving...) - if err != nil { - return "", err - } - if version != "" { - return version, nil - } - } - - // check for inherited dependencies - for _, dep := range directDependencies(pom) { - depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) - depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) - if depGroupID == groupID && depArtifactID == artifactID { - version = r.getPropertyValue(ctx, root, dep.Version) - if version != "" { - return version, nil - } - } - } - - return "", nil -} - -// resolveLicenses search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. -func (r *mavenResolver) resolveLicenses(ctx context.Context, groupID, artifactID, version string) ([]string, error) { - pom, err := r.findPom(ctx, groupID, artifactID, version) - if pom == nil || err != nil { - return nil, err - } - return r.findLicenses(ctx, pom) -} - -// Search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. -func (r *mavenResolver) findLicenses(ctx context.Context, pom *gopom.Project, processing ...mavenID) ([]string, error) { - id := makeID(pom) - if slices.Contains(processing, id) { - return nil, fmt.Errorf("cycle detected resolving licenses for: %v", id) - } - if len(processing) > r.cfg.MaxParentRecursiveDepth { - return nil, fmt.Errorf("maximum parent recursive depth (%v) reached: %v", r.cfg.MaxParentRecursiveDepth, processing) - } - - licenses := directLicenses(pom) - if len(licenses) > 0 { - return licenses, nil - } - - parent, err := r.findParent(ctx, pom) - if err != nil { - return nil, err - } - if parent == nil { - return nil, nil - } - return r.findLicenses(ctx, parent, append(processing, id)...) -} - -// func pomDescriptions(ids []mavenID) []string { -// var out []string -// for _, id := range ids { -// out = append(out, id.String()) -// } -// return out -//} - -func makeID(pom *gopom.Project) mavenID { - if pom == nil { - return mavenID{} - } - return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) -} - -// directLicenses returns the licenses defined directly in the pom -func directLicenses(pom *gopom.Project) []string { - var licenses []string - for _, license := range deref(pom.Licenses) { - if license.Name != nil { - licenses = append(licenses, *license.Name) - } else if license.URL != nil { - licenses = append(licenses, *license.URL) - } - } - return licenses -} - -// directDependencies returns all direct dependencies in a project, including all defined in profiles -func directDependencies(pom *gopom.Project) []gopom.Dependency { - dependencies := deref(pom.Dependencies) - for _, profile := range deref(pom.Profiles) { - dependencies = append(dependencies, deref(profile.Dependencies)...) - } - return dependencies -} - -// directManagedDependencies returns all managed dependencies in a project, including all defined in profiles -func directManagedDependencies(pom *gopom.Project) []gopom.Dependency { - var dependencies []gopom.Dependency - if pom.DependencyManagement != nil { - dependencies = append(dependencies, deref(pom.DependencyManagement.Dependencies)...) - } - for _, profile := range deref(pom.Profiles) { - if profile.DependencyManagement != nil { - dependencies = append(dependencies, deref(profile.DependencyManagement.Dependencies)...) - } - } - return dependencies -} - -// deref dereferences ptr if not nil, or returns the type default value if ptr is nil -func deref[T any](ptr *T) T { - if ptr == nil { - var t T - return t - } - return *ptr -} diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index 38f73b8e467..cfd004054b3 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" "time" @@ -186,3 +187,163 @@ func (r *mavenResolver) findParent(ctx context.Context, pom *gopom.Project) (*go version := r.getPropertyValue(ctx, &pomWithoutParent, parent.Version) return r.findPom(ctx, groupID, artifactID, version) } + +// Try to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies +// +//nolint:gocognit +func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Project, pom *gopom.Project, groupID, artifactID string, resolving ...mavenID) (string, error) { + id := newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) + if len(resolving) >= r.cfg.MaxParentRecursiveDepth { + return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) + } + if slices.Contains(resolving, id) { + return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) + } + resolving = append(resolving, id) + + var err error + var version string + + // check for entries in dependencyManagement first + for _, dep := range directManagedDependencies(pom) { + depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) + depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) + if depGroupID == groupID && depArtifactID == artifactID { + version = r.getPropertyValue(ctx, root, dep.Version) + if version != "" { + return version, nil + } + } + + // imported pom files should be treated just like parent poms, they are used to define versions of dependencies + if deref(dep.Type) == "pom" && deref(dep.Scope) == "import" { + depVersion := r.getPropertyValue(ctx, root, dep.Version) + + depPom, err := r.findPom(ctx, depGroupID, depArtifactID, depVersion) + if err != nil { + return "", err + } + version, err = r.findInheritedVersion(ctx, root, depPom, groupID, artifactID, resolving...) + if err != nil { + return "", err + } + if version != "" { + return version, nil + } + } + } + + // recursively check parents + parent, err := r.findParent(ctx, pom) + if err != nil { + return "", err + } + if parent != nil { + version, err = r.findInheritedVersion(ctx, root, parent, groupID, artifactID, resolving...) + if err != nil { + return "", err + } + if version != "" { + return version, nil + } + } + + // check for inherited dependencies + for _, dep := range directDependencies(pom) { + depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) + depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) + if depGroupID == groupID && depArtifactID == artifactID { + version = r.getPropertyValue(ctx, root, dep.Version) + if version != "" { + return version, nil + } + } + } + + return "", nil +} + +// resolveLicenses search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. +func (r *mavenResolver) resolveLicenses(ctx context.Context, groupID, artifactID, version string) ([]string, error) { + pom, err := r.findPom(ctx, groupID, artifactID, version) + if pom == nil || err != nil { + return nil, err + } + return r.findLicenses(ctx, pom) +} + +// Search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. +func (r *mavenResolver) findLicenses(ctx context.Context, pom *gopom.Project, processing ...mavenID) ([]string, error) { + id := makeID(pom) + if slices.Contains(processing, id) { + return nil, fmt.Errorf("cycle detected resolving licenses for: %v", id) + } + if len(processing) > r.cfg.MaxParentRecursiveDepth { + return nil, fmt.Errorf("maximum parent recursive depth (%v) reached: %v", r.cfg.MaxParentRecursiveDepth, processing) + } + + licenses := directLicenses(pom) + if len(licenses) > 0 { + return licenses, nil + } + + parent, err := r.findParent(ctx, pom) + if err != nil { + return nil, err + } + if parent == nil { + return nil, nil + } + return r.findLicenses(ctx, parent, append(processing, id)...) +} + +// func pomDescriptions(ids []mavenID) []string { +// var out []string +// for _, id := range ids { +// out = append(out, id.String()) +// } +// return out +//} + +func makeID(pom *gopom.Project) mavenID { + if pom == nil { + return mavenID{} + } + return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) +} + +// directLicenses returns the licenses defined directly in the pom +func directLicenses(pom *gopom.Project) []string { + var licenses []string + for _, license := range deref(pom.Licenses) { + if license.Name != nil { + licenses = append(licenses, *license.Name) + } else if license.URL != nil { + licenses = append(licenses, *license.URL) + } + } + return licenses +} + +// directDependencies returns all direct dependencies in a project, including all defined in profiles +func directDependencies(pom *gopom.Project) []gopom.Dependency { + dependencies := deref(pom.Dependencies) + for _, profile := range deref(pom.Profiles) { + dependencies = append(dependencies, deref(profile.Dependencies)...) + } + return dependencies +} + +// directManagedDependencies returns all managed dependencies in a project, including all defined in profiles +func directManagedDependencies(pom *gopom.Project) []gopom.Dependency { + var dependencies []gopom.Dependency + if pom.DependencyManagement != nil { + dependencies = append(dependencies, deref(pom.DependencyManagement.Dependencies)...) + } + for _, profile := range deref(pom.Profiles) { + if profile.DependencyManagement != nil { + dependencies = append(dependencies, deref(profile.DependencyManagement.Dependencies)...) + } + } + return dependencies +} diff --git a/syft/pkg/cataloger/java/maven_utils.go b/syft/pkg/cataloger/java/maven_utils.go index ee4e93f8f6e..101b74212a4 100644 --- a/syft/pkg/cataloger/java/maven_utils.go +++ b/syft/pkg/cataloger/java/maven_utils.go @@ -2,9 +2,12 @@ package java import ( "encoding/xml" + "fmt" "io" + "net/url" "os" "path/filepath" + "strings" "github.com/mitchellh/go-homedir" @@ -45,3 +48,27 @@ func getSettingsXMLLocalRepository(settingsXML io.Reader) string { } return s.LocalRepository } + +// deref dereferences ptr if not nil, or returns the type default value if ptr is nil +func deref[T any](ptr *T) T { + if ptr == nil { + var t T + return t + } + return *ptr +} + +// remotePomURL returns a URL to download a POM from a remote repository +func remotePomURL(repoURL, groupID, artifactID, version string) (requestURL string, err error) { + // groupID needs to go from maven.org -> maven/org + urlPath := strings.Split(groupID, ".") + artifactPom := fmt.Sprintf("%s-%s.pom", artifactID, version) + urlPath = append(urlPath, artifactID, version, artifactPom) + + // ex: https://repo1.maven.org/maven2/groupID/artifactID/artifactPom + requestURL, err = url.JoinPath(repoURL, urlPath...) + if err != nil { + return requestURL, fmt.Errorf("could not construct maven url: %w", err) + } + return requestURL, err +} diff --git a/syft/pkg/cataloger/java/maven_utils_test.go b/syft/pkg/cataloger/java/maven_utils_test.go index 47028d9f147..8d599b501af 100644 --- a/syft/pkg/cataloger/java/maven_utils_test.go +++ b/syft/pkg/cataloger/java/maven_utils_test.go @@ -75,3 +75,29 @@ func Test_getSettingsXmlLocalRepository(t *testing.T) { }) } } + +func Test_remotePomURL(t *testing.T) { + tests := []struct { + name string + groupID string + artifactID string + version string + expected string + }{ + { + name: "formatMavenURL correctly assembles the pom URL", + groupID: "org.springframework.boot", + artifactID: "spring-boot-starter-test", + version: "3.1.5", + expected: "https://repo1.maven.org/maven2/org/springframework/boot/spring-boot-starter-test/3.1.5/spring-boot-starter-test-3.1.5.pom", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + requestURL, err := remotePomURL(mavenBaseURL, tc.groupID, tc.artifactID, tc.version) + require.NoError(t, err, "expected no err; got %w", err) + require.Equal(t, tc.expected, requestURL) + }) + } +} From 8bcd53cfd476f558ed95f21c2d43b4652941fd5c Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 13:17:54 -0400 Subject: [PATCH 25/42] chore: cache maven pom files directly Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/maven_resolver.go | 57 +++++++++++++++++------ 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index cfd004054b3..c0eb2a42110 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -1,9 +1,11 @@ package java import ( + "bytes" "context" "errors" "fmt" + "io" "net/http" "os" "path/filepath" @@ -42,7 +44,7 @@ func newMavenID(groupID, artifactID, version *string) mavenID { type mavenResolver struct { cfg ArchiveCatalogerConfig // resolver file.Resolver - cache cache.Resolver[*gopom.Project] + cache cache.Cache resolved map[mavenID]*gopom.Project remoteRequestTimeout time.Duration checkedLocalRepo bool @@ -51,7 +53,7 @@ type mavenResolver struct { func newMavenResolver(cfg ArchiveCatalogerConfig) mavenResolver { return mavenResolver{ cfg: cfg, - cache: cache.GetResolver[*gopom.Project]("java/maven/pom", "v1"), + cache: cache.GetManager().GetCache("java/maven/repo", "v1"), resolved: map[mavenID]*gopom.Project{}, remoteRequestTimeout: time.Second * 10, } @@ -128,16 +130,19 @@ func (r *mavenResolver) findPomInRemoteRepo(ctx context.Context, groupID, artifa return nil, fmt.Errorf("missing/incomplete maven artifact coordinates -- groupId: '%s' artifactId: '%s', version: '%s'", groupID, artifactID, version) } - key := fmt.Sprintf("%s:%s:%s", groupID, artifactID, version) + requestURL, err := r.remotePomURL(groupID, artifactID, version) + if err != nil { + return nil, fmt.Errorf("unable to find pom in remote due to: %w", err) + } // Downloading snapshots requires additional steps to determine the latest snapshot version. // See: https://maven.apache.org/ref/3-LATEST/maven-repository-metadata/ if strings.HasSuffix(version, "-SNAPSHOT") { - return nil, fmt.Errorf("downloading snapshot artifacts is not supported, got: %s", key) + return nil, fmt.Errorf("downloading snapshot artifacts is not supported, got: %s", requestURL) } - return r.cache.Resolve(key, func() (*gopom.Project, error) { - requestURL, err := r.remotePomURL(groupID, artifactID, version) + cacheKey := strings.TrimPrefix(strings.TrimPrefix(requestURL, "http://"), "https://") + reader, err := r.cacheResolveReader(cacheKey, func() (io.ReadCloser, error) { if err != nil { return nil, err } @@ -158,17 +163,43 @@ func (r *mavenResolver) findPomInRemoteRepo(ctx context.Context, groupID, artifa if err != nil { return nil, fmt.Errorf("unable to get pom from Maven repository %v: %w", requestURL, err) } - defer internal.CloseAndLogError(resp.Body, requestURL) if resp.StatusCode == http.StatusNotFound { return nil, fmt.Errorf("pom not found in Maven repository at: %v", requestURL) } - - pom, err := decodePomXML(resp.Body) - if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven repository url %v: %w", requestURL, err) - } - return pom, nil + return resp.Body, err }) + if err != nil { + return nil, err + } + if reader, ok := reader.(io.Closer); ok { + defer internal.CloseAndLogError(reader, requestURL) + } + pom, err := decodePomXML(reader) + if err != nil { + return nil, fmt.Errorf("unable to parse pom from Maven repository url %v: %w", requestURL, err) + } + return pom, nil +} + +func (r *mavenResolver) cacheResolveReader(key string, resolve func() (io.ReadCloser, error)) (io.Reader, error) { + reader, err := r.cache.Read(key) + if err == nil && reader != nil { + return reader, err + } + + contentReader, err := resolve() + if err != nil { + return nil, err + } + defer internal.CloseAndLogError(contentReader, key) + + // store the contents to return a new reader with the same content + contents, err := io.ReadAll(contentReader) + if err != nil { + return nil, err + } + err = r.cache.Write(key, bytes.NewBuffer(contents)) + return bytes.NewBuffer(contents), err } func (r *mavenResolver) remotePomURL(groupID, artifactID, version string) (requestURL string, err error) { From 9864efe0738ffab995d58dcbfa84d3217359de27 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 21:15:25 -0400 Subject: [PATCH 26/42] chore: refactor parsedPomProject Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/archive_parser.go | 212 ++++++++-------- .../pkg/cataloger/java/archive_parser_test.go | 54 ++-- syft/pkg/cataloger/java/maven_resolver.go | 219 +++++++++++++---- syft/pkg/cataloger/java/parse_pom_xml.go | 232 +++--------------- syft/pkg/cataloger/java/parse_pom_xml_test.go | 141 +++++------ 5 files changed, 395 insertions(+), 463 deletions(-) diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 6519ab530f0..d477de000c0 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -8,6 +8,8 @@ import ( "path" "strings" + "github.com/vifraa/gopom" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -49,7 +51,7 @@ type archiveParser struct { fileInfo archiveFilename detectNested bool cfg ArchiveCatalogerConfig - mavenResolver + maven mavenResolver } type genericArchiveParserAdapter struct { @@ -68,7 +70,7 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f if err != nil { return nil, nil, err } - return parser.parse(ctx, gap.cfg) + return parser.parse(ctx) } // uniquePkgKey creates a unique string to identify the given package. @@ -97,19 +99,19 @@ func newJavaArchiveParser(reader file.LocationReadCloser, detectNested bool, cfg } return &archiveParser{ - fileManifest: fileManifest, - location: reader.Location, - archivePath: archivePath, - contentPath: contentPath, - fileInfo: newJavaArchiveFilename(currentFilepath), - detectNested: detectNested, - cfg: cfg, - mavenResolver: newMavenResolver(cfg), + fileManifest: fileManifest, + location: reader.Location, + archivePath: archivePath, + contentPath: contentPath, + fileInfo: newJavaArchiveFilename(currentFilepath), + detectNested: detectNested, + cfg: cfg, + maven: newMavenResolver(cfg), }, cleanupFn, nil } // parse the loaded archive and return all packages found. -func (j *archiveParser) parse(ctx context.Context, cfg ArchiveCatalogerConfig) ([]pkg.Package, []artifact.Relationship, error) { +func (j *archiveParser) parse(ctx context.Context) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package var relationships []artifact.Relationship @@ -121,7 +123,7 @@ func (j *archiveParser) parse(ctx context.Context, cfg ArchiveCatalogerConfig) ( // find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg // NOTE: we cannot generate sha1 digests from packages discovered via pom.properties/pom.xml - auxPkgs, err := j.discoverPkgsFromAllMavenFiles(ctx, parentPkg, cfg) + auxPkgs, err := j.discoverPkgsFromAllMavenFiles(ctx, parentPkg) if err != nil { return nil, nil, err } @@ -230,19 +232,13 @@ func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest 3. manifest 4. filename */ - group, name, version, pomLicenses := j.discoverMainPackageNameAndVersionFromPomInfo(ctx) + group, name, version, parsedPom := j.discoverMainPackageFromPomInfo(ctx) if name == "" { name = selectName(manifest, j.fileInfo) } if version == "" { version = selectVersion(manifest, j.fileInfo) } - if len(licenses) == 0 { - // Today we don't have a way to distinguish between licenses from the manifest and licenses from the pom.xml - // until the file.Location object can support sub-paths (i.e. paths within archives, recursively; issue https://github.com/anchore/syft/issues/2211). - // Until then it's less confusing to use the licenses from the pom.xml only if the manifest did not list any. - licenses = append(licenses, pomLicenses...) - } if len(licenses) == 0 { fileLicenses, err := j.getLicenseFromFileInArchive() @@ -255,95 +251,99 @@ func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest } // If we didn't find any licenses in the archive so far, we'll try again in Maven Central using groupIDFromJavaMetadata - if len(licenses) == 0 && j.cfg.UseNetwork { - licenses = j.findLicenseFromJavaMetadata(ctx, group, name, manifest, version, licenses) + if len(licenses) == 0 { + // Today we don't have a way to distinguish between licenses from the manifest and licenses from the pom.xml + // until the file.Location object can support sub-paths (i.e. paths within archives, recursively; issue https://github.com/anchore/syft/issues/2211). + // Until then it's less confusing to use the licenses from the pom.xml only if the manifest did not list any. + licenses = j.findLicenseFromJavaMetadata(ctx, group, name, version, parsedPom, manifest) } return name, version, licenses, nil } -func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, group, name string, manifest *pkg.JavaManifest, version string, licenses []pkg.License) []pkg.License { - var groupID = group - if group == "" { - if gID := groupIDFromJavaMetadata(name, pkg.JavaArchive{Manifest: manifest}); gID != "" { +// findLicenseFromJavaMetadata attempts to find license information from all available maven metadata properties and pom info +func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID, artifactID, version string, parsedPom *parsedPomProject, manifest *pkg.JavaManifest) []pkg.License { + if groupID == "" { + if gID := groupIDFromJavaMetadata(artifactID, pkg.JavaArchive{Manifest: manifest}); gID != "" { groupID = gID } } - pomLicenses, err := j.resolveLicenses(ctx, groupID, name, version) + var err error + var pomLicenses []gopom.License + if parsedPom != nil { + pomLicenses, err = j.maven.resolveLicenses(ctx, parsedPom.project) + if err != nil { + log.Debugf("error attempting to resolve licenses for %v: %v", newMavenIDFromPom(parsedPom.project), err) + } + } if err == nil && len(pomLicenses) == 0 { + pomLicenses, err = j.maven.findLicenses(ctx, groupID, artifactID, version) + if err != nil { + log.Debugf("error attempting to resolve licenses for %v: %v", mavenID{groupID, artifactID, version}, err) + } + } + + if len(pomLicenses) == 0 { // Try removing the last part of the groupId, as sometimes it duplicates the artifactId packages := strings.Split(groupID, ".") groupID = strings.Join(packages[:len(packages)-1], ".") - pomLicenses, _ = j.resolveLicenses(ctx, groupID, name, version) + pomLicenses, _ = j.maven.findLicenses(ctx, groupID, artifactID, version) } - if len(pomLicenses) > 0 { - pkgLicenses := pkg.NewLicensesFromLocation(j.location, pomLicenses...) - if pkgLicenses != nil { - licenses = append(licenses, pkgLicenses...) - } + return toPkgLicenses(&j.location, pomLicenses) +} + +func toPkgLicenses(location *file.Location, licenses []gopom.License) []pkg.License { + var out []pkg.License + for _, license := range licenses { + out = append(out, pkg.NewLicenseFromFields(deref(license.Name), deref(license.URL), location)) } - return licenses + return out } type parsedPomProject struct { - *pkg.JavaPomProject - Licenses []pkg.License + path string + project *gopom.Project } -// Try to find artifact group, id, version and license(s) -func (j *archiveParser) discoverMainPackageNameAndVersionFromPomInfo(ctx context.Context) (group, name, version string, licenses []pkg.License) { - pomPropertyMatches := j.fileManifest.GlobMatch(false, pomPropertiesGlob) - pomMatches := j.fileManifest.GlobMatch(false, pomXMLGlob) - var pomPropertiesObject pkg.JavaPomProperties - var pomProjectObject *parsedPomProject +// discoverMainPackageFromPomInfo attempts to resolve maven groupId, artifactId, version and other info from found pom information +func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (group, name, version string, parsedPom *parsedPomProject) { + var pomProperties pkg.JavaPomProperties // Find the pom.properties/pom.xml if the names seem like a plausible match - properties, _ := pomPropertiesByParentPath(j.archivePath, j.location, pomPropertyMatches) - projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, pomMatches, j.cfg) + properties, _ := pomPropertiesByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob)) + projects, _ := pomProjectByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob)) for parentPath, propertiesObj := range properties { if artifactIDMatchesFilename(propertiesObj.ArtifactID, j.fileInfo.name) { - pomPropertiesObject = propertiesObj + pomProperties = propertiesObj if proj, exists := projects[parentPath]; exists { - pomProjectObject = proj + parsedPom = proj break } } } - group = pomPropertiesObject.GroupID - if group == "" && pomProjectObject != nil { - group = pomProjectObject.GroupID - } - name = pomPropertiesObject.ArtifactID - if name == "" && pomProjectObject != nil { - name = pomProjectObject.ArtifactID - } - version = pomPropertiesObject.Version - if version == "" && pomProjectObject != nil { - version = pomProjectObject.Version - } + group = pomProperties.GroupID + name = pomProperties.ArtifactID + version = pomProperties.Version - if pomProjectObject == nil { - // If we have no pom.xml, check maven central using pom.properties - parentLicenses, _ := j.resolveLicenses(ctx, pomPropertiesObject.GroupID, pomPropertiesObject.ArtifactID, pomPropertiesObject.Version) - if len(parentLicenses) > 0 { - for _, licenseName := range parentLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) - } + if parsedPom != nil && parsedPom.project != nil { + pom := parsedPom.project + if group == "" { + group = j.maven.getPropertyValue(ctx, pom, pom.GroupID) + } + if name == "" { + name = j.maven.getPropertyValue(ctx, pom, pom.ArtifactID) + } + if version == "" { + version = j.maven.getPropertyValue(ctx, pom, pom.Version) } - } else { - findPomLicenses(ctx, pomProjectObject, &j.mavenResolver) - } - - if pomProjectObject != nil { - licenses = pomProjectObject.Licenses } - return group, name, version, licenses + return group, name, version, parsedPom } func artifactIDMatchesFilename(artifactID, fileName string) bool { @@ -353,28 +353,11 @@ func artifactIDMatchesFilename(artifactID, fileName string) bool { return strings.HasPrefix(artifactID, fileName) || strings.HasSuffix(fileName, artifactID) } -func findPomLicenses(ctx context.Context, pomProjectObject *parsedPomProject, r *mavenResolver) { - // If we don't have any licenses until now, and if we have a parent Pom, then we'll check the parent pom in maven central for licenses. - if pomProjectObject != nil && pomProjectObject.Parent != nil && len(pomProjectObject.Licenses) == 0 { - parentLicenses, _ := r.resolveLicenses( - ctx, - pomProjectObject.Parent.GroupID, - pomProjectObject.Parent.ArtifactID, - pomProjectObject.Parent.Version) - - if len(parentLicenses) > 0 { - for _, licenseName := range parentLicenses { - pomProjectObject.Licenses = append(pomProjectObject.Licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) - } - } - } -} - // discoverPkgsFromAllMavenFiles parses Maven POM properties/xml for a given // parent package, returning all listed Java packages found for each pom // properties discovered and potentially updating the given parentPkg with new // data. -func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, parentPkg *pkg.Package, cfg ArchiveCatalogerConfig) ([]pkg.Package, error) { +func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, error) { if parentPkg == nil { return nil, nil } @@ -388,18 +371,18 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren } // pom.xml - projects, err := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob), cfg) + projects, err := pomProjectByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob)) if err != nil { return nil, err } for parentPath, propertiesObj := range properties { - var pomProject *parsedPomProject + var parsedPom *parsedPomProject if proj, exists := projects[parentPath]; exists { - pomProject = proj + parsedPom = proj } - pkgFromPom := newPackageFromMavenData(ctx, propertiesObj, pomProject, parentPkg, j.location, &j.mavenResolver) + pkgFromPom := newPackageFromMavenData(ctx, &j.maven, propertiesObj, parsedPom, parentPkg, j.location) if pkgFromPom != nil { pkgs = append(pkgs, *pkgFromPom) } @@ -558,7 +541,7 @@ func pomPropertiesByParentPath(archivePath string, location file.Location, extra return propertiesByParentPath, nil } -func pomProjectByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string, cfg ArchiveCatalogerConfig) (map[string]*parsedPomProject, error) { +func pomProjectByParentPath(archivePath string, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) { contentsOfMavenProjectFiles, err := intFile.ContentsFromZip(archivePath, extractPaths...) if err != nil { return nil, fmt.Errorf("unable to extract maven files: %w", err) @@ -567,30 +550,26 @@ func pomProjectByParentPath(ctx context.Context, archivePath string, location fi projectByParentPath := make(map[string]*parsedPomProject) for filePath, fileContents := range contentsOfMavenProjectFiles { // TODO: when we support locations of paths within archives we should start passing the specific pom.xml location object instead of the top jar - pomProject, err := parsePomXMLProject(ctx, filePath, strings.NewReader(fileContents), location, cfg) + pom, err := decodePomXML(strings.NewReader(fileContents)) if err != nil { log.WithFields("contents-path", filePath, "location", location.Path()).Warnf("failed to parse pom.xml: %+v", err) continue } - - if pomProject == nil { + if pom == nil { continue } - // If we don't have a version, then maybe the parent pom has it... - if (pomProject.Parent == nil && pomProject.Version == "") || pomProject.ArtifactID == "" { - // TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package - continue + projectByParentPath[path.Dir(filePath)] = &parsedPomProject{ + path: filePath, + project: pom, } - - projectByParentPath[path.Dir(filePath)] = pomProject } return projectByParentPath, nil } // newPackageFromMavenData processes a single Maven POM properties for a given parent package, returning all listed Java packages found and // associating each discovered package to the given parent package. Note the pom.xml is optional, the pom.properties is not. -func newPackageFromMavenData(ctx context.Context, pomProperties pkg.JavaPomProperties, parsedPomProject *parsedPomProject, parentPkg *pkg.Package, location file.Location, r *mavenResolver) *pkg.Package { +func newPackageFromMavenData(ctx context.Context, r *mavenResolver, pomProperties pkg.JavaPomProperties, parsedPom *parsedPomProject, parentPkg *pkg.Package, location file.Location) *pkg.Package { // keep the artifact name within the virtual path if this package does not match the parent package vPathSuffix := "" groupID := "" @@ -613,23 +592,24 @@ func newPackageFromMavenData(ctx context.Context, pomProperties pkg.JavaPomPrope virtualPath := location.Path() + vPathSuffix var pkgPomProject *pkg.JavaPomProject - licenses := make([]pkg.License, 0) - if parsedPomProject == nil { + var err error + var pomLicenses []gopom.License + if parsedPom == nil { // If we have no pom.xml, check maven central using pom.properties - parentLicenses, _ := r.resolveLicenses(ctx, pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version) - if len(parentLicenses) > 0 { - for _, licenseName := range parentLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) - } - } + pomLicenses, err = r.findLicenses(ctx, pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version) } else { - findPomLicenses(ctx, parsedPomProject, r) + pkgPomProject = newPomProject(ctx, r, parsedPom.path, parsedPom.project) + pomLicenses, err = r.resolveLicenses(ctx, parsedPom.project) } - if parsedPomProject != nil { - pkgPomProject = parsedPomProject.JavaPomProject - licenses = append(licenses, parsedPomProject.Licenses...) + if err != nil { + log.Debugf("error while attempting to resolve licenses for %v: %v", mavenID{pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version}, err) + } + + licenses := make([]pkg.License, 0) + for _, license := range pomLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(deref(license.Name), deref(license.URL), &location)) } p := pkg.Package{ diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index a5cadd16d1a..ab6ddbbaafc 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -19,6 +19,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vifraa/gopom" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -89,8 +90,11 @@ func TestSearchMavenForLicenses(t *testing.T) { }, expectedLicenses: []pkg.License{ { - Type: license.Declared, - Value: `The Apache Software License, Version 2.0`, + Type: license.Declared, + Value: `The Apache Software License, Version 2.0`, + URLs: []string{ + "http://www.apache.org/licenses/LICENSE-2.0.txt", + }, SPDXExpression: ``, }, }, @@ -116,8 +120,9 @@ func TestSearchMavenForLicenses(t *testing.T) { defer cleanupFn() // assert licenses are discovered from upstream - _, _, _, licenses := ap.discoverMainPackageNameAndVersionFromPomInfo(context.Background()) - assert.Equal(t, tc.expectedLicenses, licenses) + _, _, _, parsedPom := ap.discoverMainPackageFromPomInfo(context.Background()) + licenses, _ := ap.maven.resolveLicenses(context.Background(), parsedPom.project) + assert.Equal(t, tc.expectedLicenses, toPkgLicenses(nil, licenses)) }) } } @@ -387,7 +392,7 @@ func TestParseJar(t *testing.T) { defer cleanupFn() require.NoError(t, err) - actual, _, err := parser.parse(context.Background(), cfg) + actual, _, err := parser.parse(context.Background()) require.NoError(t, err) if len(actual) != len(test.expected) { @@ -799,26 +804,23 @@ func Test_newPackageFromMavenData(t *testing.T) { Version: "1.0", }, project: &parsedPomProject{ - JavaPomProject: &pkg.JavaPomProject{ - Parent: &pkg.JavaPomParent{ - GroupID: "some-parent-group-id", - ArtifactID: "some-parent-artifact-id", - Version: "1.0-parent", + project: &gopom.Project{ + Parent: &gopom.Parent{ + GroupID: ptr("some-parent-group-id"), + ArtifactID: ptr("some-parent-artifact-id"), + Version: ptr("1.0-parent"), }, - Name: "some-name", - GroupID: "some-group-id", - ArtifactID: "some-artifact-id", - Version: "1.0", - Description: "desc", - URL: "aweso.me", - }, - Licenses: []pkg.License{ - { - Value: "MIT", - SPDXExpression: "MIT", - Type: license.Declared, - URLs: []string{"https://opensource.org/licenses/MIT"}, - Locations: file.NewLocationSet(file.NewLocation("some-license-path")), + Name: ptr("some-name"), + GroupID: ptr("some-group-id"), + ArtifactID: ptr("some-artifact-id"), + Version: ptr("1.0"), + Description: ptr("desc"), + URL: ptr("aweso.me"), + Licenses: &[]gopom.License{ + { + Name: ptr("MIT"), + URL: ptr("https://opensource.org/licenses/MIT"), + }, }, }, }, @@ -854,7 +856,7 @@ func Test_newPackageFromMavenData(t *testing.T) { SPDXExpression: "MIT", Type: license.Declared, URLs: []string{"https://opensource.org/licenses/MIT"}, - Locations: file.NewLocationSet(file.NewLocation("some-license-path")), + Locations: file.NewLocationSet(file.NewLocation("given/virtual/path")), }, ), Metadata: pkg.JavaArchive{ @@ -1079,7 +1081,7 @@ func Test_newPackageFromMavenData(t *testing.T) { test.expectedParent.Locations = locations r := newMavenResolver(DefaultArchiveCatalogerConfig()) - actualPackage := newPackageFromMavenData(context.Background(), test.props, test.project, test.parent, file.NewLocation(virtualPath), &r) + actualPackage := newPackageFromMavenData(context.Background(), &r, test.props, test.project, test.parent, file.NewLocation(virtualPath)) if test.expectedPackage == nil { require.Nil(t, actualPackage) } else { diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index c0eb2a42110..ce005ebfff1 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "reflect" "slices" "strings" "time" @@ -27,10 +28,7 @@ type mavenID struct { Version string } -func (m mavenID) String() string { - return fmt.Sprintf("%s:%s:%s", m.GroupID, m.ArtifactID, m.Version) -} - +// newMavenID is a convenience to construct a mavenId from string pointers, used by gopom func newMavenID(groupID, artifactID, version *string) mavenID { return mavenID{ GroupID: deref(groupID), @@ -39,6 +37,23 @@ func newMavenID(groupID, artifactID, version *string) mavenID { } } +// newMavenIDFromPom creates a new mavenID from a pom +func newMavenIDFromPom(pom *gopom.Project) mavenID { + if pom == nil { + return mavenID{} + } + return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) +} + +// Valid indicates this mavenID has non-empty groupId, artifactId, and version +func (m mavenID) Valid() bool { + return m.GroupID != "" && m.ArtifactID != "" && m.Version != "" +} + +func (m mavenID) String() string { + return fmt.Sprintf("groupId: %s artifactId:%s version:%s", m.GroupID, m.ArtifactID, m.Version) +} + // mavenResolver is a short-lived utility to resolve maven poms from multiple sources, including: // the scanned filesystem, local maven cache directories, remote maven repositories, and the syft cache type mavenResolver struct { @@ -59,13 +74,134 @@ func newMavenResolver(cfg ArchiveCatalogerConfig) mavenResolver { } } +// getPropertyValue gets property values by emulating maven property resolution logic, looking in the project's variables +// as well as supporting the project expressions like ${project.parent.groupId}. +// Properties which are not resolved result in empty string "" +// +//nolint:gocognit +func (r *mavenResolver) getPropertyValue(ctx context.Context, pom *gopom.Project, propertyValue *string) string { + if propertyValue == nil { + return "" + } + resolved, err := r.resolveExpression(ctx, pom, *propertyValue, nil) + if err != nil { + log.Debugf("error resolving maven property: %s: %v", *propertyValue, err) + return "" + } + return resolved +} + +// resolveExpression resolves an expression, which may be a plain string or a string with ${ property.references } +// +//nolint:gocognit +func (r *mavenResolver) resolveExpression(ctx context.Context, pom *gopom.Project, expression string, resolving []string) (string, error) { + var err error + return expressionMatcher.ReplaceAllStringFunc(expression, func(match string) string { + propertyExpression := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } + resolved, e := r.resolveProperty(ctx, pom, propertyExpression, resolving) + if e != nil { + err = errors.Join(err, e) + return "" + } + return resolved + }), err +} + +// resolveProperty resolves properties recursively from the root project +// +//nolint:gocognit +func (r *mavenResolver) resolveProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { + // prevent cycles + if slices.Contains(resolving, propertyExpression) { + return "", fmt.Errorf("cycle detected resolving: %s", propertyExpression) + } + resolving = append(resolving, propertyExpression) + + value, err := r.resolveProjectProperty(ctx, pom, propertyExpression, resolving) + if err != nil { + return value, err + } + if value != "" { + return value, nil + } + + current := pom + for current != nil { + if current.Properties != nil && current.Properties.Entries != nil { + if value, ok := current.Properties.Entries[propertyExpression]; ok { + return r.resolveExpression(ctx, pom, value, resolving) // property values can contain expressions + } + } + current, err = r.resolveParent(ctx, current) + if err != nil { + return "", err + } + } + + return "", fmt.Errorf("unable to resolve property: %s", propertyExpression) +} + +// resolveProjectProperty resolves properties on the project +// +//nolint:gocognit +func (r *mavenResolver) resolveProjectProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { + // see if we have a project.x expression and process this based + // on the xml tags in gopom + parts := strings.Split(propertyExpression, ".") + numParts := len(parts) + if numParts > 1 && strings.TrimSpace(parts[0]) == "project" { + pomValue := reflect.ValueOf(pom).Elem() + pomValueType := pomValue.Type() + for partNum := 1; partNum < numParts; partNum++ { + if pomValueType.Kind() != reflect.Struct { + break + } + + part := parts[partNum] + for fieldNum := 0; fieldNum < pomValueType.NumField(); fieldNum++ { + f := pomValueType.Field(fieldNum) + tag := f.Tag.Get("xml") + tag = strings.Split(tag, ",")[0] + // a segment of the property name matches the xml tag for the field, + // so we need to recurse down the nested structs or return a match + // if we're done. + if part != tag { + continue + } + + pomValue = pomValue.Field(fieldNum) + pomValueType = pomValue.Type() + if pomValueType.Kind() == reflect.Ptr { + // we were recursing down the nested structs, but one of the steps + // we need to take is a nil pointer, so give up + if pomValue.IsNil() { + return "", fmt.Errorf("property undefined: %s", propertyExpression) + } + pomValue = pomValue.Elem() + if !pomValue.IsZero() { + // we found a non-zero value whose tag matches this part of the property name + pomValueType = pomValue.Type() + } + } + // If this was the last part of the property name, return the value + if partNum == numParts-1 { + value := fmt.Sprintf("%v", pomValue.Interface()) + return r.resolveExpression(ctx, pom, value, resolving) + } + break + } + } + } + return "", nil +} + // findPom gets a pom from cache, local repository, or downloads from a remote Maven repository depending on configuration func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { if groupID == "" || artifactID == "" || version == "" { - return nil, fmt.Errorf("invalid parent specification, require non-empty values for groupID :'%s', artifactID :'%s', version :'%s'", groupID, artifactID, version) + return nil, fmt.Errorf("invalid maven pom specification, require non-empty values for groupID: '%s', artifactID: '%s', version: '%s'", groupID, artifactID, version) } - id := newMavenID(&groupID, &artifactID, &version) + id := mavenID{groupID, artifactID, version} pom := r.resolved[id] if pom != nil { @@ -86,7 +222,7 @@ func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, versio // resolve via network maven repository if pom == nil && r.cfg.UseNetwork { - pom, err := r.findPomInRemoteRepo(ctx, groupID, artifactID, version) + pom, err := r.findPomInRemoteRepository(ctx, groupID, artifactID, version) if pom != nil { r.resolved[id] = pom return pom, nil @@ -124,13 +260,13 @@ func (r *mavenResolver) findPomInLocalRepository(groupID, artifactID, version st return decodePomXML(pomFile) } -// Download the pom file from a (remote) Maven repository over HTTP. -func (r *mavenResolver) findPomInRemoteRepo(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { +// findPomInRemoteRepository download the pom file from a (remote) Maven repository over HTTP +func (r *mavenResolver) findPomInRemoteRepository(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { if groupID == "" || artifactID == "" || version == "" { return nil, fmt.Errorf("missing/incomplete maven artifact coordinates -- groupId: '%s' artifactId: '%s', version: '%s'", groupID, artifactID, version) } - requestURL, err := r.remotePomURL(groupID, artifactID, version) + requestURL, err := remotePomURL(r.cfg.MavenBaseURL, groupID, artifactID, version) if err != nil { return nil, fmt.Errorf("unable to find pom in remote due to: %w", err) } @@ -202,11 +338,8 @@ func (r *mavenResolver) cacheResolveReader(key string, resolve func() (io.ReadCl return bytes.NewBuffer(contents), err } -func (r *mavenResolver) remotePomURL(groupID, artifactID, version string) (requestURL string, err error) { - return remotePomURL(r.cfg.MavenBaseURL, groupID, artifactID, version) -} - -func (r *mavenResolver) findParent(ctx context.Context, pom *gopom.Project) (*gopom.Project, error) { +// resolveParent attempts to resolve +func (r *mavenResolver) resolveParent(ctx context.Context, pom *gopom.Project) (*gopom.Project, error) { if pom == nil || pom.Parent == nil { return nil, nil } @@ -265,7 +398,7 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr } // recursively check parents - parent, err := r.findParent(ctx, pom) + parent, err := r.resolveParent(ctx, pom) if err != nil { return "", err } @@ -294,18 +427,18 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr return "", nil } -// resolveLicenses search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. -func (r *mavenResolver) resolveLicenses(ctx context.Context, groupID, artifactID, version string) ([]string, error) { +// findLicenses search pom for license, traversing parent poms if needed +func (r *mavenResolver) findLicenses(ctx context.Context, groupID, artifactID, version string) ([]gopom.License, error) { pom, err := r.findPom(ctx, groupID, artifactID, version) if pom == nil || err != nil { return nil, err } - return r.findLicenses(ctx, pom) + return r.resolveLicenses(ctx, pom) } -// Search pom for license, traversing parent poms if needed. Also returns if a pom file was found in order to differentiate between no pom and no license found. -func (r *mavenResolver) findLicenses(ctx context.Context, pom *gopom.Project, processing ...mavenID) ([]string, error) { - id := makeID(pom) +// resolveLicenses searches the pom for license, traversing parent poms if needed +func (r *mavenResolver) resolveLicenses(ctx context.Context, pom *gopom.Project, processing ...mavenID) ([]gopom.License, error) { + id := newMavenIDFromPom(pom) if slices.Contains(processing, id) { return nil, fmt.Errorf("cycle detected resolving licenses for: %v", id) } @@ -313,47 +446,33 @@ func (r *mavenResolver) findLicenses(ctx context.Context, pom *gopom.Project, pr return nil, fmt.Errorf("maximum parent recursive depth (%v) reached: %v", r.cfg.MaxParentRecursiveDepth, processing) } - licenses := directLicenses(pom) - if len(licenses) > 0 { - return licenses, nil + directLicenses := r.directLicenses(ctx, pom) + if len(directLicenses) > 0 { + return directLicenses, nil } - parent, err := r.findParent(ctx, pom) + parent, err := r.resolveParent(ctx, pom) if err != nil { return nil, err } if parent == nil { return nil, nil } - return r.findLicenses(ctx, parent, append(processing, id)...) -} - -// func pomDescriptions(ids []mavenID) []string { -// var out []string -// for _, id := range ids { -// out = append(out, id.String()) -// } -// return out -//} - -func makeID(pom *gopom.Project) mavenID { - if pom == nil { - return mavenID{} - } - return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) + return r.resolveLicenses(ctx, parent, append(processing, id)...) } -// directLicenses returns the licenses defined directly in the pom -func directLicenses(pom *gopom.Project) []string { - var licenses []string +// directLicenses appends the directly specified licenses with non-empty name or url +func (r *mavenResolver) directLicenses(ctx context.Context, pom *gopom.Project) []gopom.License { + var out []gopom.License for _, license := range deref(pom.Licenses) { - if license.Name != nil { - licenses = append(licenses, *license.Name) - } else if license.URL != nil { - licenses = append(licenses, *license.URL) + // if we find non-empty licenses, return them + name := r.getPropertyValue(ctx, pom, license.Name) + url := r.getPropertyValue(ctx, pom, license.URL) + if name != "" || url != "" { + out = append(out, license) } } - return licenses + return out } // directDependencies returns all direct dependencies in a project, including all defined in profiles diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index d3ce52011ab..92b583b3856 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -7,9 +7,7 @@ import ( "errors" "fmt" "io" - "reflect" "regexp" - "slices" "strings" "github.com/saintfish/chardet" @@ -36,12 +34,12 @@ func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, _ file.R r := newMavenResolver(gap.cfg) var pkgs []pkg.Package - for _, dep := range directDependencies(pom) { id := newMavenID(dep.GroupID, dep.ArtifactID, dep.Version) - log.Debugf("add dependency to SBOM : [%v]", id) - p, err := r.newPackageFromDependency( + log.Tracef("adding dependency to SBOM: %v", id) + p, err := newPackageFromDependency( ctx, + &r, pom, dep, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), @@ -58,58 +56,25 @@ func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, _ file.R return pkgs, nil, nil } -func parsePomXMLProject(ctx context.Context, path string, reader io.Reader, location file.Location, cfg ArchiveCatalogerConfig) (*parsedPomProject, error) { - pom, err := decodePomXML(reader) - if err != nil { - return nil, err - } - - resolver := newMavenResolver(cfg) - - return resolver.newPomProject(ctx, path, pom, location), nil -} - -func (r *mavenResolver) newPomProject(ctx context.Context, path string, pom *gopom.Project, location file.Location) *parsedPomProject { - artifactID := deref(pom.ArtifactID) - name := deref(pom.Name) - projectURL := deref(pom.URL) - - var licenses []pkg.License - if pom.Licenses != nil { - for _, license := range *pom.Licenses { - var licenseName, licenseURL string - if license.Name != nil { - licenseName = *license.Name - } - if license.URL != nil { - licenseURL = *license.URL - } - - if licenseName == "" && licenseURL == "" { - continue - } - - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, licenseURL, &location)) - } - } +func newPomProject(ctx context.Context, r *mavenResolver, path string, pom *gopom.Project) *pkg.JavaPomProject { + artifactID := r.getPropertyValue(ctx, pom, pom.ArtifactID) + name := r.getPropertyValue(ctx, pom, pom.Name) + projectURL := r.getPropertyValue(ctx, pom, pom.URL) log.WithFields("path", path, "artifactID", artifactID, "name", name, "projectURL", projectURL).Trace("parsing pom.xml") - return &parsedPomProject{ - JavaPomProject: &pkg.JavaPomProject{ - Path: path, - Parent: r.pomParent(ctx, pom), - GroupID: r.getPropertyValue(ctx, pom, pom.GroupID), - ArtifactID: artifactID, - Version: r.getPropertyValue(ctx, pom, pom.Version), - Name: name, - Description: cleanDescription(pom.Description), - URL: projectURL, - }, - Licenses: licenses, + return &pkg.JavaPomProject{ + Path: path, + Parent: pomParent(ctx, r, pom), + GroupID: r.getPropertyValue(ctx, pom, pom.GroupID), + ArtifactID: artifactID, + Version: r.getPropertyValue(ctx, pom, pom.Version), + Name: name, + Description: cleanDescription(r.getPropertyValue(ctx, pom, pom.Description)), + URL: projectURL, } } -func (r *mavenResolver) newPackageFromDependency(ctx context.Context, pom *gopom.Project, dep gopom.Dependency, locations ...file.Location) (*pkg.Package, error) { +func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom.Project, dep gopom.Dependency, locations ...file.Location) (*pkg.Package, error) { groupID := r.getPropertyValue(ctx, pom, dep.GroupID) artifactID := r.getPropertyValue(ctx, pom, dep.ArtifactID) version := r.getPropertyValue(ctx, pom, dep.Version) @@ -127,7 +92,7 @@ func (r *mavenResolver) newPackageFromDependency(ctx context.Context, pom *gopom }, } - licenses := make([]pkg.License, 0) + var licenses []pkg.License if version == "" { dependencyPom, depErr := r.findPom(ctx, groupID, artifactID, version) if depErr != nil { @@ -135,9 +100,9 @@ func (r *mavenResolver) newPackageFromDependency(ctx context.Context, pom *gopom err = errors.Join(err, depErr) } if dependencyPom != nil { - parentLicenses, _ := r.findLicenses(ctx, dependencyPom) - for _, licenseName := range parentLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) + depLicenses, _ := r.resolveLicenses(ctx, dependencyPom) + for _, license := range depLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(deref(license.Name), deref(license.URL), nil)) } } } @@ -211,30 +176,28 @@ func getUtf8Reader(content io.Reader) (io.Reader, error) { return inputReader, nil } -func (r *mavenResolver) pomParent(ctx context.Context, pom *gopom.Project) *pkg.JavaPomParent { +func pomParent(ctx context.Context, r *mavenResolver, pom *gopom.Project) *pkg.JavaPomParent { if pom == nil || pom.Parent == nil { return nil } - groupID := deref(pom.Parent.GroupID) - artifactID := deref(pom.Parent.ArtifactID) - version := deref(pom.Parent.Version) + groupID := r.getPropertyValue(ctx, pom, pom.Parent.GroupID) + artifactID := r.getPropertyValue(ctx, pom, pom.Parent.ArtifactID) + version := r.getPropertyValue(ctx, pom, pom.Parent.Version) + if groupID == "" && artifactID == "" && version == "" { return nil } return &pkg.JavaPomParent{ - GroupID: r.getPropertyValue(ctx, pom, pom.Parent.GroupID), + GroupID: groupID, ArtifactID: artifactID, - Version: r.getPropertyValue(ctx, pom, pom.Parent.Version), + Version: version, } } -func cleanDescription(original *string) (cleaned string) { - if original == nil { - return "" - } - descriptionLines := strings.Split(*original, "\n") +func cleanDescription(original string) (cleaned string) { + descriptionLines := strings.Split(original, "\n") for _, line := range descriptionLines { line = strings.TrimSpace(line) if len(line) == 0 { @@ -244,138 +207,3 @@ func cleanDescription(original *string) (cleaned string) { } return strings.TrimSpace(cleaned) } - -// getPropertyValue gets property values by emulating maven property resolution logic, looking in the project's variables -// as well as supporting the project expressions like ${project.parent.groupId}. -// Properties which are not resolved result in empty string "" -// -//nolint:gocognit -func (r *mavenResolver) getPropertyValue(ctx context.Context, pom *gopom.Project, propertyValue *string) string { - if propertyValue == nil { - return "" - } - resolved, err := r.resolveExpression(ctx, pom, *propertyValue, nil) - if err != nil { - log.Debugf("error resolving maven property: %s: %v", *propertyValue, err) - return "" - } - return resolved -} - -// resolveExpression resolves an expression, which may be a plain string or a string with ${ property.references } -// -//nolint:gocognit -func (r *mavenResolver) resolveExpression(ctx context.Context, pom *gopom.Project, expression string, resolving []string) (string, error) { - var err error - return expressionMatcher.ReplaceAllStringFunc(expression, func(match string) string { - propertyExpression := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } - resolved, e := r.resolveProperty(ctx, pom, propertyExpression, resolving) - if e != nil { - err = errors.Join(err, e) - return "" - } - return resolved - }), err -} - -// resolveProperty resolves properties recursively from the root project -// -//nolint:gocognit -func (r *mavenResolver) resolveProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { - // prevent cycles - if slices.Contains(resolving, propertyExpression) { - return "", fmt.Errorf("cycle detected resolving: %s", propertyExpression) - } - resolving = append(resolving, propertyExpression) - - value, err := r.resolveProjectProperty(ctx, pom, propertyExpression, resolving) - if err != nil { - return value, err - } - if value != "" { - return value, nil - } - - current := pom - for current != nil { - if current.Properties != nil && current.Properties.Entries != nil { - if value, ok := current.Properties.Entries[propertyExpression]; ok { - return r.resolveExpression(ctx, pom, value, resolving) // property values can contain expressions - } - } - current, err = r.findParent(ctx, current) - if err != nil { - return "", err - } - } - - return "", fmt.Errorf("unable to resolve property: %s", propertyExpression) -} - -// resolveProjectProperty resolves properties on the project -// -//nolint:gocognit -func (r *mavenResolver) resolveProjectProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { - // see if we have a project.x expression and process this based - // on the xml tags in gopom - parts := strings.Split(propertyExpression, ".") - numParts := len(parts) - if numParts > 1 && strings.TrimSpace(parts[0]) == "project" { - pomValue := reflect.ValueOf(pom).Elem() - pomValueType := pomValue.Type() - for partNum := 1; partNum < numParts; partNum++ { - if pomValueType.Kind() != reflect.Struct { - break - } - - part := parts[partNum] - for fieldNum := 0; fieldNum < pomValueType.NumField(); fieldNum++ { - f := pomValueType.Field(fieldNum) - tag := f.Tag.Get("xml") - tag = strings.Split(tag, ",")[0] - // a segment of the property name matches the xml tag for the field, - // so we need to recurse down the nested structs or return a match - // if we're done. - if part != tag { - continue - } - - pomValue = pomValue.Field(fieldNum) - pomValueType = pomValue.Type() - if pomValueType.Kind() == reflect.Ptr { - // we were recursing down the nested structs, but one of the steps - // we need to take is a nil pointer, so give up - if pomValue.IsNil() { - return "", fmt.Errorf("property undefined: %s", propertyExpression) - } - pomValue = pomValue.Elem() - if !pomValue.IsZero() { - // we found a non-zero value whose tag matches this part of the property name - pomValueType = pomValue.Type() - } - } - // If this was the last part of the property name, return the value - if partNum == numParts-1 { - value := fmt.Sprintf("%v", pomValue.Interface()) - return r.resolveExpression(ctx, pom, value, resolving) - } - break - } - } - } - return "", nil -} - -// func pomProperties(p gopom.Project) map[string]string { -// if p.Properties != nil { -// return p.Properties.Entries -// } -// return map[string]string{} -//} - -// func deref(s *string) string { -// if s == nil { -// return "" -// } -// return *s -//} diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index efc28c5c09b..8ccbbfa9ce2 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -266,63 +266,60 @@ func Test_parsePomXMLProject(t *testing.T) { jarLocation := file.NewLocation("path/to/archive.jar") tests := []struct { name string - expected parsedPomProject + project *pkg.JavaPomProject + licenses []pkg.License }{ { name: "go case", - expected: parsedPomProject{ - JavaPomProject: &pkg.JavaPomProject{ - Path: "test-fixtures/pom/commons-codec.pom.xml", - Parent: &pkg.JavaPomParent{ - GroupID: "org.apache.commons", - ArtifactID: "commons-parent", - Version: "42", - }, - GroupID: "commons-codec", - ArtifactID: "commons-codec", - Version: "1.11", - Name: "Apache Commons Codec", - Description: "The Apache Commons Codec package contains simple encoder and decoders for various formats such as Base64 and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.", - URL: "http://commons.apache.org/proper/commons-codec/", - }, + project: &pkg.JavaPomProject{ + Path: "test-fixtures/pom/commons-codec.pom.xml", + Parent: &pkg.JavaPomParent{ + GroupID: "org.apache.commons", + ArtifactID: "commons-parent", + Version: "42", + }, + GroupID: "commons-codec", + ArtifactID: "commons-codec", + Version: "1.11", + Name: "Apache Commons Codec", + Description: "The Apache Commons Codec package contains simple encoder and decoders for various formats such as Base64 and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.", + URL: "http://commons.apache.org/proper/commons-codec/", }, }, { name: "with license data", - expected: parsedPomProject{ - JavaPomProject: &pkg.JavaPomProject{ - Path: "test-fixtures/pom/neo4j-license-maven-plugin.pom.xml", - Parent: &pkg.JavaPomParent{ - GroupID: "org.sonatype.oss", - ArtifactID: "oss-parent", - Version: "7", - }, - GroupID: "org.neo4j.build.plugins", - ArtifactID: "license-maven-plugin", - Version: "4-SNAPSHOT", - Name: "${project.artifactId}", // TODO: this is not an ideal answer - Description: "Maven 2 plugin to check and update license headers in source files", - URL: "http://components.neo4j.org/${project.artifactId}/${project.version}", // TODO: this is not an ideal answer - }, - Licenses: []pkg.License{ - { - Value: "The Apache Software License, Version 2.0", - SPDXExpression: "", // TODO: ideally we would parse this title to get Apache-2.0 (created issue #2210 https://github.com/anchore/syft/issues/2210) - Type: license.Declared, - URLs: []string{"http://www.apache.org/licenses/LICENSE-2.0.txt"}, - Locations: file.NewLocationSet(jarLocation), - }, - { - Value: "MIT", - SPDXExpression: "MIT", - Type: license.Declared, - Locations: file.NewLocationSet(jarLocation), - }, - { - Type: license.Declared, - URLs: []string{"https://opensource.org/license/unlicense/"}, - Locations: file.NewLocationSet(jarLocation), - }, + project: &pkg.JavaPomProject{ + Path: "test-fixtures/pom/neo4j-license-maven-plugin.pom.xml", + Parent: &pkg.JavaPomParent{ + GroupID: "org.sonatype.oss", + ArtifactID: "oss-parent", + Version: "7", + }, + GroupID: "org.neo4j.build.plugins", + ArtifactID: "license-maven-plugin", + Version: "4-SNAPSHOT", + Name: "license-maven-plugin", + Description: "Maven 2 plugin to check and update license headers in source files", + URL: "http://components.neo4j.org/license-maven-plugin/4-SNAPSHOT", + }, + licenses: []pkg.License{ + { + Value: "The Apache Software License, Version 2.0", + SPDXExpression: "", // TODO: ideally we would parse this title to get Apache-2.0 (created issue #2210 https://github.com/anchore/syft/issues/2210) + Type: license.Declared, + URLs: []string{"http://www.apache.org/licenses/LICENSE-2.0.txt"}, + Locations: file.NewLocationSet(jarLocation), + }, + { + Value: "MIT", + SPDXExpression: "MIT", + Type: license.Declared, + Locations: file.NewLocationSet(jarLocation), + }, + { + Type: license.Declared, + URLs: []string{"https://opensource.org/license/unlicense/"}, + Locations: file.NewLocationSet(jarLocation), }, }, }, @@ -330,14 +327,20 @@ func Test_parsePomXMLProject(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fixture, err := os.Open(test.expected.Path) + fixture, err := os.Open(test.project.Path) assert.NoError(t, err) - cfg := ArchiveCatalogerConfig{} + r := newMavenResolver(ArchiveCatalogerConfig{}) + + pom, err := gopom.ParseFromReader(fixture) + require.NoError(t, err) - actual, err := parsePomXMLProject(context.Background(), fixture.Name(), fixture, jarLocation, cfg) + actual := newPomProject(context.Background(), &r, fixture.Name(), pom) assert.NoError(t, err) + assert.Equal(t, test.project, actual) - assert.Equal(t, &test.expected, actual) + licenses := r.directLicenses(context.Background(), pom) + assert.NoError(t, err) + assert.Equal(t, test.licenses, toPkgLicenses(&jarLocation, licenses)) }) } } @@ -351,7 +354,7 @@ func Test_pomParent(t *testing.T) { { name: "only group ID", input: &gopom.Parent{ - GroupID: stringPointer("org.something"), + GroupID: ptr("org.something"), }, expected: &pkg.JavaPomParent{ GroupID: "org.something", @@ -360,7 +363,7 @@ func Test_pomParent(t *testing.T) { { name: "only artifact ID", input: &gopom.Parent{ - ArtifactID: stringPointer("something"), + ArtifactID: ptr("something"), }, expected: &pkg.JavaPomParent{ ArtifactID: "something", @@ -369,7 +372,7 @@ func Test_pomParent(t *testing.T) { { name: "only Version", input: &gopom.Parent{ - Version: stringPointer("something"), + Version: ptr("something"), }, expected: &pkg.JavaPomParent{ Version: "something", @@ -388,7 +391,7 @@ func Test_pomParent(t *testing.T) { { name: "unused field", input: &gopom.Parent{ - RelativePath: stringPointer("something"), + RelativePath: ptr("something"), }, expected: nil, }, @@ -397,7 +400,7 @@ func Test_pomParent(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := newMavenResolver(DefaultArchiveCatalogerConfig()) - assert.Equal(t, test.expected, r.pomParent(context.Background(), &gopom.Project{Parent: test.input})) + assert.Equal(t, test.expected, pomParent(context.Background(), &r, &gopom.Project{Parent: test.input})) }) } } @@ -420,7 +423,7 @@ func Test_cleanDescription(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, cleanDescription(stringPointer(test.input))) + assert.Equal(t, test.expected, cleanDescription(test.input)) }) } } @@ -448,7 +451,7 @@ func Test_resolveProperty(t *testing.T) { name: "groupId", property: "${project.groupId}", pom: gopom.Project{ - GroupID: stringPointer("org.some.group"), + GroupID: ptr("org.some.group"), }, expected: "org.some.group", }, @@ -457,7 +460,7 @@ func Test_resolveProperty(t *testing.T) { property: "${project.parent.groupId}", pom: gopom.Project{ Parent: &gopom.Parent{ - GroupID: stringPointer("org.some.parent"), + GroupID: ptr("org.some.parent"), }, }, expected: "org.some.parent", @@ -485,7 +488,7 @@ func Test_resolveProperty(t *testing.T) { property: "${springboot.version}", pom: gopom.Project{ Parent: &gopom.Parent{ - Version: stringPointer("1.2.3"), + Version: ptr("1.2.3"), }, Properties: &gopom.Properties{ Entries: map[string]string{ @@ -500,7 +503,7 @@ func Test_resolveProperty(t *testing.T) { property: "${springboot.version}", pom: gopom.Project{ Parent: &gopom.Parent{ - Version: stringPointer("1.2.3"), + Version: ptr("1.2.3"), }, }, expected: "", @@ -510,7 +513,7 @@ func Test_resolveProperty(t *testing.T) { property: "${springboot.version}", pom: gopom.Project{ Parent: &gopom.Parent{ - Version: stringPointer("${undefined.version}"), + Version: ptr("${undefined.version}"), }, Properties: &gopom.Properties{ Entries: map[string]string{ @@ -551,7 +554,7 @@ func Test_resolveProperty(t *testing.T) { property: "${cyclic.version}", pom: gopom.Project{ Parent: &gopom.Parent{ - Version: stringPointer("${cyclic.version}"), + Version: ptr("${cyclic.version}"), }, Properties: &gopom.Properties{ Entries: map[string]string{ @@ -568,14 +571,14 @@ func Test_resolveProperty(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := newMavenResolver(DefaultArchiveCatalogerConfig()) - resolved := r.getPropertyValue(context.Background(), &test.pom, stringPointer(test.property)) + resolved := r.getPropertyValue(context.Background(), &test.pom, ptr(test.property)) assert.Equal(t, test.expected, resolved) }) } } -func stringPointer(s string) *string { - return &s +func ptr[T any](value T) *T { + return &value } func Test_getUtf8Reader(t *testing.T) { From b51c5eb158f40e39312adb01671939e79737c8c7 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 21:22:22 -0400 Subject: [PATCH 27/42] chore: fix env var names Signed-off-by: Keith Zantow --- cmd/syft/internal/options/java.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index 979d647df88..89b0399df3b 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -7,8 +7,8 @@ import ( type javaConfig struct { UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` - UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` - MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` + UseMavenLocalRepository bool `yaml:"use-maven-local-repository" json:"use-maven-local-repository" mapstructure:"use-maven-local-repository"` + MavenLocalRepositoryDir string `yaml:"maven-local-repository-dir" json:"maven-local-repository-dir" mapstructure:"maven-local-repository-dir"` MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"` MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` } From a3485b3121aeb10321772e69c17afc30645ef376 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 21:27:33 -0400 Subject: [PATCH 28/42] chore: update some comments Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/maven_resolver.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index ce005ebfff1..c0b02774789 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -195,7 +195,7 @@ func (r *mavenResolver) resolveProjectProperty(ctx context.Context, pom *gopom.P return "", nil } -// findPom gets a pom from cache, local repository, or downloads from a remote Maven repository depending on configuration +// findPom gets a pom from cache, local repository, or from a remote Maven repository depending on configuration func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { if groupID == "" || artifactID == "" || version == "" { return nil, fmt.Errorf("invalid maven pom specification, require non-empty values for groupID: '%s', artifactID: '%s', version: '%s'", groupID, artifactID, version) @@ -237,8 +237,7 @@ func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, versio return nil, nil } -// Try to get the Pom from the users local repository in the users home dir. -// Returns (nil, false) when file cannot be found or read for any reason. +// findPomInLocalRepository attempts to get the POM from the users local maven repository func (r *mavenResolver) findPomInLocalRepository(groupID, artifactID, version string) (*gopom.Project, error) { groupPath := filepath.Join(strings.Split(groupID, ".")...) pomFilePath := filepath.Join(r.cfg.MavenLocalRepositoryDir, groupPath, artifactID, version, artifactID+"-"+version+".pom") From f11cb4943799f71c9164b8c2d6f916b6e8f602ae Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 21:38:52 -0400 Subject: [PATCH 29/42] chore: maven_resolver refactoring Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/maven_resolver.go | 3 + .../pkg/cataloger/java/maven_resolver_test.go | 150 ++++++++++++++++++ syft/pkg/cataloger/java/parse_pom_xml.go | 3 - syft/pkg/cataloger/java/parse_pom_xml_test.go | 149 ----------------- 4 files changed, 153 insertions(+), 152 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index c0b02774789..bcfbef2029e 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "slices" "strings" "time" @@ -54,6 +55,8 @@ func (m mavenID) String() string { return fmt.Sprintf("groupId: %s artifactId:%s version:%s", m.GroupID, m.ArtifactID, m.Version) } +var expressionMatcher = regexp.MustCompile("[$][{][^}]+[}]") + // mavenResolver is a short-lived utility to resolve maven poms from multiple sources, including: // the scanned filesystem, local maven cache directories, remote maven repositories, and the syft cache type mavenResolver struct { diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index 39427027516..97d4e0888a8 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -10,8 +10,158 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/stretchr/testify/require" + "github.com/vifraa/gopom" ) +func Test_resolveProperty(t *testing.T) { + tests := []struct { + name string + property string + pom gopom.Project + expected string + }{ + { + name: "property", + property: "${version.number}", + pom: gopom.Project{ + Properties: &gopom.Properties{ + Entries: map[string]string{ + "version.number": "12.5.0", + }, + }, + }, + expected: "12.5.0", + }, + { + name: "groupId", + property: "${project.groupId}", + pom: gopom.Project{ + GroupID: ptr("org.some.group"), + }, + expected: "org.some.group", + }, + { + name: "parent groupId", + property: "${project.parent.groupId}", + pom: gopom.Project{ + Parent: &gopom.Parent{ + GroupID: ptr("org.some.parent"), + }, + }, + expected: "org.some.parent", + }, + { + name: "nil pointer halts search", + property: "${project.parent.groupId}", + pom: gopom.Project{ + Parent: nil, + }, + expected: "", + }, + { + name: "nil string pointer halts search", + property: "${project.parent.groupId}", + pom: gopom.Project{ + Parent: &gopom.Parent{ + GroupID: nil, + }, + }, + expected: "", + }, + { + name: "double dereference", + property: "${springboot.version}", + pom: gopom.Project{ + Parent: &gopom.Parent{ + Version: ptr("1.2.3"), + }, + Properties: &gopom.Properties{ + Entries: map[string]string{ + "springboot.version": "${project.parent.version}", + }, + }, + }, + expected: "1.2.3", + }, + { + name: "map missing stops double dereference", + property: "${springboot.version}", + pom: gopom.Project{ + Parent: &gopom.Parent{ + Version: ptr("1.2.3"), + }, + }, + expected: "", + }, + { + name: "resolution halts even if it resolves to a variable", + property: "${springboot.version}", + pom: gopom.Project{ + Parent: &gopom.Parent{ + Version: ptr("${undefined.version}"), + }, + Properties: &gopom.Properties{ + Entries: map[string]string{ + "springboot.version": "${project.parent.version}", + }, + }, + }, + expected: "", + }, + { + name: "resolution halts even if cyclic", + property: "${springboot.version}", + pom: gopom.Project{ + Properties: &gopom.Properties{ + Entries: map[string]string{ + "springboot.version": "${springboot.version}", + }, + }, + }, + expected: "", + }, + { + name: "resolution halts even if cyclic more steps", + property: "${cyclic.version}", + pom: gopom.Project{ + Properties: &gopom.Properties{ + Entries: map[string]string{ + "other.version": "${cyclic.version}", + "springboot.version": "${other.version}", + "cyclic.version": "${springboot.version}", + }, + }, + }, + expected: "", + }, + { + name: "resolution halts even if cyclic involving parent", + property: "${cyclic.version}", + pom: gopom.Project{ + Parent: &gopom.Parent{ + Version: ptr("${cyclic.version}"), + }, + Properties: &gopom.Properties{ + Entries: map[string]string{ + "other.version": "${parent.version}", + "springboot.version": "${other.version}", + "cyclic.version": "${springboot.version}", + }, + }, + }, + expected: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := newMavenResolver(DefaultArchiveCatalogerConfig()) + resolved := r.getPropertyValue(context.Background(), &test.pom, ptr(test.property)) + require.Equal(t, test.expected, resolved) + }) + } +} + func Test_mavenResolverLocal(t *testing.T) { dir, err := filepath.Abs("test-fixtures/pom/maven-repo") require.NoError(t, err) diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 92b583b3856..8154f4ed01f 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "regexp" "strings" "github.com/saintfish/chardet" @@ -23,8 +22,6 @@ import ( const pomXMLGlob = "*pom.xml" -var expressionMatcher = regexp.MustCompile("[$][{][^}]+[}]") - func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) if err != nil || pom == nil { diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 8ccbbfa9ce2..d2e355a380c 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -428,155 +428,6 @@ func Test_cleanDescription(t *testing.T) { } } -func Test_resolveProperty(t *testing.T) { - tests := []struct { - name string - property string - pom gopom.Project - expected string - }{ - { - name: "property", - property: "${version.number}", - pom: gopom.Project{ - Properties: &gopom.Properties{ - Entries: map[string]string{ - "version.number": "12.5.0", - }, - }, - }, - expected: "12.5.0", - }, - { - name: "groupId", - property: "${project.groupId}", - pom: gopom.Project{ - GroupID: ptr("org.some.group"), - }, - expected: "org.some.group", - }, - { - name: "parent groupId", - property: "${project.parent.groupId}", - pom: gopom.Project{ - Parent: &gopom.Parent{ - GroupID: ptr("org.some.parent"), - }, - }, - expected: "org.some.parent", - }, - { - name: "nil pointer halts search", - property: "${project.parent.groupId}", - pom: gopom.Project{ - Parent: nil, - }, - expected: "", - }, - { - name: "nil string pointer halts search", - property: "${project.parent.groupId}", - pom: gopom.Project{ - Parent: &gopom.Parent{ - GroupID: nil, - }, - }, - expected: "", - }, - { - name: "double dereference", - property: "${springboot.version}", - pom: gopom.Project{ - Parent: &gopom.Parent{ - Version: ptr("1.2.3"), - }, - Properties: &gopom.Properties{ - Entries: map[string]string{ - "springboot.version": "${project.parent.version}", - }, - }, - }, - expected: "1.2.3", - }, - { - name: "map missing stops double dereference", - property: "${springboot.version}", - pom: gopom.Project{ - Parent: &gopom.Parent{ - Version: ptr("1.2.3"), - }, - }, - expected: "", - }, - { - name: "resolution halts even if it resolves to a variable", - property: "${springboot.version}", - pom: gopom.Project{ - Parent: &gopom.Parent{ - Version: ptr("${undefined.version}"), - }, - Properties: &gopom.Properties{ - Entries: map[string]string{ - "springboot.version": "${project.parent.version}", - }, - }, - }, - expected: "", - }, - { - name: "resolution halts even if cyclic", - property: "${springboot.version}", - pom: gopom.Project{ - Properties: &gopom.Properties{ - Entries: map[string]string{ - "springboot.version": "${springboot.version}", - }, - }, - }, - expected: "", - }, - { - name: "resolution halts even if cyclic more steps", - property: "${cyclic.version}", - pom: gopom.Project{ - Properties: &gopom.Properties{ - Entries: map[string]string{ - "other.version": "${cyclic.version}", - "springboot.version": "${other.version}", - "cyclic.version": "${springboot.version}", - }, - }, - }, - expected: "", - }, - { - name: "resolution halts even if cyclic involving parent", - property: "${cyclic.version}", - pom: gopom.Project{ - Parent: &gopom.Parent{ - Version: ptr("${cyclic.version}"), - }, - Properties: &gopom.Properties{ - Entries: map[string]string{ - "other.version": "${parent.version}", - "springboot.version": "${other.version}", - "cyclic.version": "${springboot.version}", - }, - }, - }, - expected: "", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - r := newMavenResolver(DefaultArchiveCatalogerConfig()) - resolved := r.getPropertyValue(context.Background(), &test.pom, ptr(test.property)) - assert.Equal(t, test.expected, resolved) - }) - } -} - func ptr[T any](value T) *T { return &value } From 1960f704dbac82c48a791e4470ffd0b721fe224a Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 17 Jul 2024 21:47:57 -0400 Subject: [PATCH 30/42] chore: trim test files Signed-off-by: Keith Zantow --- .../commons-parent/54/commons-parent-54.pom | 1858 +---------------- .../junit/junit-bom/5.9.0/junit-bom-5.9.0.pom | 8 - .../junit/junit-bom/5.9.1/junit-bom-5.9.1.pom | 7 - .../3.4.6/opensaml-parent-3.4.6.pom | 81 - 4 files changed, 1 insertion(+), 1953 deletions(-) diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom index 3d4bdb8d499..3fd66f094f4 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/apache/commons/commons-parent/54/commons-parent-54.pom @@ -1,21 +1,7 @@ @@ -31,97 +17,24 @@ pom Apache Commons Parent The Apache Commons Parent POM provides common settings for all Apache Commons components. - 2006 https://commons.apache.org/commons-parent-pom.html - - - 3.3.9 - - - 2022-09-18T13:49:41Z ${project.version} - RC1 - COMMONSSITE - - 53 true Gary Gregory 86fdc7e2a11262cb - - - 1.3 1.3 - - false - 1.22 - 1.0 3.4.2 3.3.0 @@ -134,69 +47,12 @@ 4.3.0 EpochMillis 2.7.1 - 0.5.5 - 3.0.0-M7 - 5.1.8 - 0.8.8 - 0.16.0 - 3.3.0 - 3.4.1 - 3.3.0 - 3.19.0 - 6.49.0 - 3.4.1 - 0.15 - 1.8.0 - 1.1 - 3.1.0 - 3.0.0 - 6.3.1 5.9.0 - - 3.12.1 3.2.1 4.7.2.0 - 4.7.2 - 3.0.0-M7 - 3.0.0-M7 3.5.2 @@ -221,22 +77,8 @@ 0.90 false - ${project.artifactId} - ${project.artifactId} @@ -250,180 +92,16 @@ ${project.build.directory}/osgi/MANIFEST.MF - scp - iso-8859-1 - ${commons.encoding} - ${commons.encoding} - ${commons.encoding} - - - https://docs.oracle.com/javase/6/docs/api/ - https://docs.oracle.com/javase/7/docs/api/ - https://docs.oracle.com/javase/8/docs/api/ - https://docs.oracle.com/javase/9/docs/api/ - https://docs.oracle.com/javase/10/docs/api/ - https://docs.oracle.com/en/java/javase/11/docs/api/ - https://docs.oracle.com/en/java/javase/12/docs/api/ - https://docs.oracle.com/en/java/javase/13/docs/api/ - https://docs.oracle.com/en/java/javase/14/docs/api/ - https://docs.oracle.com/en/java/javase/15/docs/api/ - https://docs.oracle.com/en/java/javase/16/docs/api/ - https://docs.oracle.com/en/java/javase/17/docs/api/ - https://docs.oracle.com/en/java/javase/18/docs/api/ - - ${commons.javadoc7.java.link} - - https://docs.oracle.com/javaee/5/api/ - https://docs.oracle.com/javaee/6/api/ - https://docs.oracle.com/javaee/7/api/ - - ${commons.javadoc.javaee6.link} - - yyyy-MM-dd HH:mm:ssZ ${scmBranch}@r${buildNumber}; ${maven.build.timestamp} - - - info - - - 100 - - - false - - - false - - 100 - - false - - - ${user.home}/commons-sites - - ${commons.componentid} - - https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-${commons.componentid} - ${commons.site.cache}/${commons.site.path} - commons.site - - - true - false - false - - - scm:svn:https://dist.apache.org/repos/dist/dev/commons/${commons.componentid} - - - ${user.name} - DEADBEEF - - https://analysis.apache.org/ - - - . - RELEASE-NOTES.txt - - - - - - - Commons User List - user-subscribe@commons.apache.org - user-unsubscribe@commons.apache.org - user@commons.apache.org - https://mail-archives.apache.org/mod_mbox/commons-user/ - - https://markmail.org/list/org.apache.commons.users/ - https://www.mail-archive.com/user@commons.apache.org/ - - - - Commons Dev List - dev-subscribe@commons.apache.org - dev-unsubscribe@commons.apache.org - dev@commons.apache.org - https://mail-archives.apache.org/mod_mbox/commons-dev/ - - https://markmail.org/list/org.apache.commons.dev/ - https://www.mail-archive.com/dev@commons.apache.org/ - - - - Commons Issues List - issues-subscribe@commons.apache.org - issues-unsubscribe@commons.apache.org - https://mail-archives.apache.org/mod_mbox/commons-issues/ - - https://markmail.org/list/org.apache.commons.issues/ - https://www.mail-archive.com/issues@commons.apache.org/ - - - - Commons Commits List - commits-subscribe@commons.apache.org - commits-unsubscribe@commons.apache.org - https://mail-archives.apache.org/mod_mbox/commons-commits/ - - https://markmail.org/list/org.apache.commons.commits/ - https://www.mail-archive.com/commits@commons.apache.org/ - - - - Apache Announce List - announce-subscribe@apache.org - announce-unsubscribe@apache.org - https://mail-archives.apache.org/mod_mbox/www-announce/ - - https://markmail.org/list/org.apache.announce/ - https://www.mail-archive.com/announce@apache.org/ - - - - - - - scm:git:http://gitbox.apache.org/repos/asf/commons-parent.git - scm:git:https://gitbox.apache.org/repos/asf/commons-parent.git - https://gitbox.apache.org/repos/asf?p=commons-parent.git - - - - jira - https://issues.apache.org/jira/browse/COMMONSSITE - - - - GitHub - https://github.com/apache/commons-parent/actions - - @@ -436,1492 +114,7 @@ - - - clean apache-rat:check package site - - - - src/main/resources - - - - ${basedir} - META-INF - - NOTICE.txt - LICENSE.txt - NOTICE - LICENSE - - - - - - - - src/test/resources - - - - ${basedir} - META-INF - - NOTICE.txt - LICENSE.txt - NOTICE - LICENSE - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${commons.compiler.version} - - ${maven.compiler.source} - ${maven.compiler.target} - ${commons.encoding} - - ${commons.compiler.fork} - - ${commons.compiler.compilerVersion} - ${commons.compiler.javac} - - - - org.apache.maven.plugins - maven-assembly-plugin - ${commons.assembly-plugin.version} - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${commons.javadoc.version} - - - true - ${maven.compiler.source} - ${commons.compiler.javadoc} - ${commons.encoding} - ${commons.docEncoding} - - true - true - - ${commons.javadoc.java.link} - ${commons.javadoc.javaee.link} - - - - true - true - - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - - - true - - - - - org.apache.maven.plugins - maven-site-plugin - ${commons.site-plugin.version} - - - true - - - - - org.apache.maven.wagon - wagon-ssh - ${commons.wagon-ssh.version} - - - - - attach-descriptor - - attach-descriptor - - - - - - org.apache.maven.plugins - maven-source-plugin - ${commons.source-plugin.version} - - - - true - true - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${commons.surefire.version} - - - org.apache.maven.plugins - maven-failsafe-plugin - ${commons.failsafe.version} - - - - com.github.siom79.japicmp - japicmp-maven-plugin - ${commons.japicmp.version} - - - - ${project.groupId} - ${project.artifactId} - ${commons.bc.version} - jar - - - - - ${project.build.directory}/${project.artifactId}-${project.version}.jar - - - - true - ${commons.japicmp.breakBuildOnBinaryIncompatibleModifications} - ${commons.japicmp.breakBuildOnSourceIncompatibleModifications} - - true - true - true - ${commons.japicmp.ignoreMissingClasses} - - - METHOD_NEW_DEFAULT - true - true - PATCH - - - - - - - org.apache.commons - commons-build-plugin - ${commons.build-plugin.version} - - ${commons.release.name} - - - - org.apache.commons - commons-release-plugin - ${commons.release-plugin.version} - - - org.apache.felix - maven-bundle-plugin - ${commons.felix.version} - true - - - - biz.aQute.bnd - biz.aQute.bndlib - ${commons.biz.aQute.bndlib.version} - - - - - org.apache.rat - apache-rat-plugin - ${commons.rat.version} - - - org.codehaus.mojo - build-helper-maven-plugin - ${commons.build-helper.version} - - - org.codehaus.mojo - buildnumber-maven-plugin - ${commons.buildnumber-plugin.version} - - - org.codehaus.mojo - versions-maven-plugin - - 2.12.0 - - - org.jacoco - jacoco-maven-plugin - ${commons.jacoco.version} - - - - prepare-agent - process-test-classes - - prepare-agent - - - - report - site - - report - - - - check - - check - - - - - BUNDLE - - - CLASS - COVEREDRATIO - ${commons.jacoco.classRatio} - - - INSTRUCTION - COVEREDRATIO - ${commons.jacoco.instructionRatio} - - - METHOD - COVEREDRATIO - ${commons.jacoco.methodRatio} - - - BRANCH - COVEREDRATIO - ${commons.jacoco.branchRatio} - - - LINE - COVEREDRATIO - ${commons.jacoco.lineRatio} - - - COMPLEXITY - COVEREDRATIO - ${commons.jacoco.complexityRatio} - - - - - ${commons.jacoco.haltOnFailure} - - - - - - org.apache.maven.plugins - maven-project-info-reports-plugin - ${commons.project-info.version} - - - - org.apache.bcel - bcel - 6.5.0 - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${commons.checkstyle-plugin.version} - - - com.puppycrawl.tools - checkstyle - ${commons.checkstyle.version} - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${commons.spotbugs.plugin.version} - - - com.github.spotbugs - spotbugs - ${commons.spotbugs.impl.version} - - - - - org.apache.maven.plugins - maven-pmd-plugin - ${commons.pmd.version} - - - net.sourceforge.pmd - pmd-core - ${commons.pmd-impl.version} - - - net.sourceforge.pmd - pmd-java - ${commons.pmd-impl.version} - - - net.sourceforge.pmd - pmd-javascript - ${commons.pmd-impl.version} - - - net.sourceforge.pmd - pmd-jsp - ${commons.pmd-impl.version} - - - - - org.cyclonedx - cyclonedx-maven-plugin - ${commons.cyclonedx.version} - - - package - - makeAggregateBom - - - - - library - 1.4 - true - true - true - true - true - false - false - true - all - ${project.artifactId}-${project.version}-bom - - - - org.spdx - spdx-maven-plugin - ${commons.spdx.version} - - - build-spdx - - createSPDX - - - - - - *.spdx - - - - - org.codehaus.mojo - javancss-maven-plugin - - - - - - - **/*.java - - - - - - - - - - maven-assembly-plugin - - - src/assembly/src.xml - - gnu - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - javadoc.resources - generate-sources - - run - - - - - - - - - - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - true - org.apache.maven.plugins - maven-enforcer-plugin - ${commons.enforcer-plugin.version} - - - - 3.5.0 - - - ${maven.compiler.target} - - - true - - - - enforce-maven-3 - - enforce - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${commons.jar-plugin.version} - - - - test-jar - - - - true - - - - - - ${commons.manifestfile} - - ${project.name} - ${project.version} - ${project.organization.name} - ${project.name} - ${project.version} - ${project.organization.name} - org.apache - ${implementation.build} - ${maven.compiler.source} - ${maven.compiler.target} - - - - - - maven-source-plugin - - - create-source-jar - - jar-no-fork - test-jar-no-fork - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${commons.surefire.java} - - - - - org.apache.commons - commons-build-plugin - - - org.apache.felix - maven-bundle-plugin - - - - true - - ${commons.osgi.excludeDependencies} - ${project.build.directory}/osgi - - - <_nouses>true - - <_removeheaders>JAVA_1_3_HOME,JAVA_1_4_HOME,JAVA_1_5_HOME,JAVA_1_6_HOME,JAVA_1_7_HOME,JAVA_1_8_HOME,JAVA_1_9_HOME - ${commons.osgi.symbolicName} - ${commons.osgi.export} - ${commons.osgi.private} - ${commons.osgi.import} - ${commons.osgi.dynamicImport} - ${project.url} - - - - - bundle-manifest - process-classes - - manifest - - - - - - - org.apache.rat - apache-rat-plugin - - - - - site-content/** - .checkstyle - .fbprefs - .pmd - .asf.yaml - src/site/resources/download_*.cgi - src/site/resources/profile.* - profile.* - - maven-eclipse.xml - .externalToolBuilders/** - - .vscode/** - - - - - rat-check - validate - - check - - - - - - - org.apache.maven.plugins - maven-scm-publish-plugin - - ${project.reporting.outputDirectory} - scm:svn:${commons.scmPubUrl} - ${commons.scmPubCheckoutDirectory} - ${commons.scmPubServer} - true - - - - scm-publish - site-deploy - - publish-scm - - - - - - org.codehaus.mojo - versions-maven-plugin - - - org.cyclonedx - cyclonedx-maven-plugin - - - org.spdx - spdx-maven-plugin - - - - - - - - - - org.apache.maven.plugins - maven-changes-plugin - ${commons.changes.version} - - ${basedir}/src/changes/changes.xml - Fix Version,Key,Component,Summary,Type,Resolution,Status - - Fix Version DESC,Type,Key DESC - Fixed - Resolved,Closed - - Bug,New Feature,Task,Improvement,Wish,Test - - true - ${commons.changes.onlyCurrentVersion} - ${commons.changes.maxEntries} - ${commons.changes.runOnlyAtExecutionRoot} - - - - - changes-report - jira-report - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${commons.javadoc.version} - - - - default - - javadoc - - - - - - org.apache.maven.plugins - maven-jxr-plugin - ${commons.jxr.version} - - - org.apache.maven.plugins - maven-project-info-reports-plugin - ${commons.project-info.version} - - - - - index - summary - modules - - team - scm - issue-management - mailing-lists - dependency-info - dependency-management - dependencies - dependency-convergence - ci-management - - - distribution-management - - - - - - org.apache.maven.plugins - maven-site-plugin - ${commons.site-plugin.version} - - - - navigation.xml,changes.xml - - - - - org.apache.maven.plugins - maven-surefire-report-plugin - ${commons.surefire-report.version} - - ${commons.surefire-report.aggregate} - - - - - org.apache.rat - apache-rat-plugin - ${commons.rat.version} - - - - - site-content/** - .checkstyle - .fbprefs - .pmd - .asf.yaml - src/site/resources/download_*.cgi - src/site/resources/profile.* - profile.* - - maven-eclipse.xml - .externalToolBuilders/** - - .vscode/** - - - - - - - - svn - - - .svn - - - - - - org.codehaus.mojo - buildnumber-maven-plugin - - - validate - - create - - - - - - true - - ?????? - - - javasvn - - - - - - - - - - module-name - - - profile.module-name - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - ${commons.module.name} - - - - - - - - - - - parse-target-version - - - - user.home - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - parse-version - - - parse-version - - - javaTarget - ${maven.compiler.target} - - - - - - - - - - - - animal-sniffer - - - - src/site/resources/profile.noanimal - - - - - - java${javaTarget.majorVersion}${javaTarget.minorVersion} - - - - - - - - org.codehaus.mojo - animal-sniffer-maven-plugin - ${commons.animal-sniffer.version} - - - checkAPIcompatibility - - - - check - - - - - - org.codehaus.mojo.signature - ${animal-sniffer.signature} - ${commons.animal-sniffer.signature.version} - - - - - - - - - - jacoco - - - - src/site/resources/profile.jacoco - - - - - - org.jacoco - jacoco-maven-plugin - ${commons.jacoco.version} - - - - - - - org.jacoco - jacoco-maven-plugin - ${commons.jacoco.version} - - - - - report - - - - - - - - - - cobertura - - - src/site/resources/profile.cobertura - - - - - - org.codehaus.mojo - cobertura-maven-plugin - ${commons.cobertura.version} - - - - - - - japicmp - - [1.8,) - - src/site/resources/profile.japicmp - - - - - - com.github.siom79.japicmp - japicmp-maven-plugin - - - verify - - cmp - - - - - - - - - - com.github.siom79.japicmp - japicmp-maven-plugin - ${commons.japicmp.version} - - - true - ${commons.japicmp.breakBuildOnBinaryIncompatibleModifications} - ${commons.japicmp.breakBuildOnSourceIncompatibleModifications} - - true - true - true - ${commons.japicmp.ignoreMissingClasses} - - - METHOD_NEW_DEFAULT - true - true - PATCH - - - - - - - - - - - - release - - - - maven-install-plugin - - true - - - - maven-release-plugin - - - -Prelease - - - - maven-javadoc-plugin - - - create-javadoc-jar - - javadoc - jar - - package - - - - ${maven.compiler.source} - ${commons.compiler.javadoc} - - - - maven-assembly-plugin - ${commons.assembly-plugin.version} - true - - - - single - - - verify - - - - - - maven-gpg-plugin - - - sign-artifacts - verify - - sign - - - - - - org.apache.commons - commons-release-plugin - - - clean-staging - clean - - clean-staging - - - - detatch-distributions - verify - - detach-distributions - - - - stage-distributions - deploy - - stage-distributions - - - - - - - - - - - apache-release - - - - maven-release-plugin - - apache-release - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-test-sources - - test-jar - - - - - - maven-install-plugin - - true - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - - - - - - java-1.7 - - true - 1.7 - ${JAVA_1_7_HOME}/bin/javac - ${JAVA_1_7_HOME}/bin/javadoc - ${JAVA_1_7_HOME}/bin/java - - - - - - java-1.8 - - true - 1.8 - ${JAVA_1_8_HOME}/bin/javac - ${JAVA_1_8_HOME}/bin/javadoc - ${JAVA_1_8_HOME}/bin/java - - - - - - java-1.9 - - true - 1.9 - ${JAVA_1_9_HOME}/bin/javac - ${JAVA_1_9_HOME}/bin/javadoc - ${JAVA_1_9_HOME}/bin/java - - - - - - java-1.10 - - true - 1.10 - ${JAVA_1_10_HOME}/bin/javac - ${JAVA_1_10_HOME}/bin/javadoc - ${JAVA_1_10_HOME}/bin/java - - - - - - java-1.11 - - true - 1.11 - ${JAVA_1_11_HOME}/bin/javac - ${JAVA_1_11_HOME}/bin/javadoc - ${JAVA_1_11_HOME}/bin/java - - - - - - java-1.12 - - true - 1.12 - ${JAVA_1_12_HOME}/bin/javac - ${JAVA_1_12_HOME}/bin/javadoc - ${JAVA_1_12_HOME}/bin/java - - - - - - java-1.13 - - true - 1.13 - ${JAVA_1_13_HOME}/bin/javac - ${JAVA_1_13_HOME}/bin/javadoc - ${JAVA_1_13_HOME}/bin/java - - - - - - - - test-deploy - - id::default::file:target/deploy - true - - - - - - release-notes - - - - org.apache.maven.plugins - maven-changes-plugin - ${commons.changes.version} - - - src/changes - true - ${changes.announcementDirectory} - ${changes.announcementFile} - - ${commons.release.version} - - - - - create-release-notes - generate-resources - - announcement-generate - - - - - - - - - - - svn-buildnumber - - - !buildNumber.skip - !true - - - - - - org.codehaus.mojo - buildnumber-maven-plugin - - - generate-resources - - create - - - - - - true - - ?????? - false - false - - - - - - - - javasvn - - - - org.codehaus.mojo - buildnumber-maven-plugin - - - javasvn - - - - - - - - - jdk7-plugin-fix-version - - [1.7,1.8) - - - 3.5.1 - 1.17 - 3.5.0 - - - - site-basic @@ -1935,54 +128,5 @@ true - - - travis-cobertura - - - - org.codehaus.mojo - cobertura-maven-plugin - ${commons.cobertura.version} - - - xml - - - - - org.eluder.coveralls - coveralls-maven-plugin - ${commons.coveralls.version} - - ${commons.coveralls.timestampFormat} - - - - - - - - travis-jacoco - - - - org.jacoco - jacoco-maven-plugin - ${commons.jacoco.version} - - - org.eluder.coveralls - coveralls-maven-plugin - ${commons.coveralls.version} - - ${commons.coveralls.timestampFormat} - - - - - - - diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom index de9e3fc418d..eb1b959bc86 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.0/junit-bom-5.9.0.pom @@ -1,19 +1,11 @@ - - - - - 4.0.0 org.junit junit-bom 5.9.0 pom - JUnit 5 (Bill of Materials) - This Bill of Materials POM can be used to ease dependency management when referencing multiple JUnit artifacts using Gradle or Maven. - https://junit.org/junit5/ Eclipse Public License v2.0 diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom index 57e6b86cbb1..c21518a394c 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/junit/junit-bom/5.9.1/junit-bom-5.9.1.pom @@ -1,19 +1,12 @@ - - - - - 4.0.0 org.junit junit-bom 5.9.1 pom JUnit 5 (Bill of Materials) - This Bill of Materials POM can be used to ease dependency management when referencing multiple JUnit artifacts using Gradle or Maven. - https://junit.org/junit5/ Eclipse Public License v2.0 diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom index e3e2be465ec..b23c101a0c7 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom @@ -10,43 +10,12 @@ OpenSAML - - A library for creating, reading, writing and performing some processing of SAML messages. - - For further information see https://wiki.shibboleth.net/confluence/display/OS30/Home - org.opensaml opensaml-parent 3.4.6 pom - - ../opensaml-core - ../opensaml-storage-api - ../opensaml-security-api - ../opensaml-xmlsec-api - ../opensaml-messaging-api - ../opensaml-soap-api - ../opensaml-saml-api - ../opensaml-xacml-api - ../opensaml-xacml-saml-api - - ../opensaml-storage-impl - ../opensaml-security-impl - ../opensaml-xmlsec-impl - ../opensaml-messaging-impl - ../opensaml-soap-impl - ../opensaml-saml-impl - ../opensaml-xacml-impl - ../opensaml-xacml-saml-impl - ../opensaml-profile-api - ../opensaml-profile-impl - - ../opensaml-bom - ../opensaml-tests-bom - - 7.5.2 5.4.2 @@ -55,11 +24,7 @@ ${opensaml-parent.site.url}${project.artifactId} - - - - net.shibboleth.utilities java-support @@ -70,11 +35,6 @@ commons-codec - - - - - net.shibboleth.utilities java-support @@ -90,12 +50,8 @@ - - javax.xml.bind jaxb-api @@ -123,11 +79,6 @@ 2.12.3 - - - - - net.shibboleth.ext spring-extensions @@ -151,36 +102,4 @@ - - - - org.apache.maven.plugins - maven-site-plugin - - - attach-descriptor - - attach-descriptor - - - - - ../opensaml-parent/src/site - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - ${automatic.module.name} - - - - - - - \ No newline at end of file From bbcf9652e60430f46632338b47b9b1f1c2f60669 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 23 Jul 2024 15:25:41 -0400 Subject: [PATCH 31/42] feat: add maven relativePath parent resolution Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/archive_parser.go | 2 +- .../pkg/cataloger/java/archive_parser_test.go | 2 +- syft/pkg/cataloger/java/config.go | 2 +- syft/pkg/cataloger/java/maven_resolver.go | 107 +++++++++++++++--- .../pkg/cataloger/java/maven_resolver_test.go | 40 ++++++- syft/pkg/cataloger/java/parse_pom_xml.go | 7 +- syft/pkg/cataloger/java/parse_pom_xml_test.go | 6 +- .../org/child-one/1.3.6/child-one-1.3.6.pom | 1 - .../parent-one/3.11.0/parent-one-3.11.0.pom | 1 - .../parent-two/13.7.8/parent-two-13.7.8.pom | 4 +- .../pom/relative/child-1/pom.xml | 42 +++++++ .../pom/relative/parent-1/pom.xml | 52 +++++++++ .../pom/relative/parent-2/pom.xml | 60 ++++++++++ 13 files changed, 295 insertions(+), 31 deletions(-) create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/relative/child-1/pom.xml create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-2/pom.xml diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index d477de000c0..c6ed0377064 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -106,7 +106,7 @@ func newJavaArchiveParser(reader file.LocationReadCloser, detectNested bool, cfg fileInfo: newJavaArchiveFilename(currentFilepath), detectNested: detectNested, cfg: cfg, - maven: newMavenResolver(cfg), + maven: newMavenResolver(nil, cfg), }, cleanupFn, nil } diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index ab6ddbbaafc..2a2f13569de 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -1080,7 +1080,7 @@ func Test_newPackageFromMavenData(t *testing.T) { } test.expectedParent.Locations = locations - r := newMavenResolver(DefaultArchiveCatalogerConfig()) + r := newMavenResolver(nil, DefaultArchiveCatalogerConfig()) actualPackage := newPackageFromMavenData(context.Background(), &r, test.props, test.project, test.parent, file.NewLocation(virtualPath)) if test.expectedPackage == nil { require.Nil(t, actualPackage) diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 63566c1d6fa..df48f12bfde 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -20,7 +20,7 @@ func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { UseMavenLocalRepository: false, MavenLocalRepositoryDir: defaultMavenLocalRepoDir(), MavenBaseURL: mavenBaseURL, - MaxParentRecursiveDepth: 5, + MaxParentRecursiveDepth: 10, } } diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index bcfbef2029e..a5f64478e36 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "os" + "path" "path/filepath" "reflect" "regexp" @@ -20,6 +21,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/cache" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" ) // mavenID is the unique identifier for a package in Maven @@ -52,7 +54,7 @@ func (m mavenID) Valid() bool { } func (m mavenID) String() string { - return fmt.Sprintf("groupId: %s artifactId:%s version:%s", m.GroupID, m.ArtifactID, m.Version) + return fmt.Sprintf("(groupId: %s artifactId: %s version: %s)", m.GroupID, m.ArtifactID, m.Version) } var expressionMatcher = regexp.MustCompile("[$][{][^}]+[}]") @@ -60,20 +62,26 @@ var expressionMatcher = regexp.MustCompile("[$][{][^}]+[}]") // mavenResolver is a short-lived utility to resolve maven poms from multiple sources, including: // the scanned filesystem, local maven cache directories, remote maven repositories, and the syft cache type mavenResolver struct { - cfg ArchiveCatalogerConfig - // resolver file.Resolver + cfg ArchiveCatalogerConfig cache cache.Cache resolved map[mavenID]*gopom.Project remoteRequestTimeout time.Duration checkedLocalRepo bool + // fileResolver and pomLocations are used to resolve parent poms by relativePath + fileResolver file.Resolver + pomLocations map[*gopom.Project]file.Location } -func newMavenResolver(cfg ArchiveCatalogerConfig) mavenResolver { +// newMavenResolver constructs a new mavenResolver with the given configuration. +// NOTE: the fileResolver is optional and if provided will be used to resolve parent poms by relative path +func newMavenResolver(fileResolver file.Resolver, cfg ArchiveCatalogerConfig) mavenResolver { return mavenResolver{ cfg: cfg, cache: cache.GetManager().GetCache("java/maven/repo", "v1"), resolved: map[mavenID]*gopom.Project{}, remoteRequestTimeout: time.Second * 10, + fileResolver: fileResolver, + pomLocations: map[*gopom.Project]file.Location{}, } } @@ -340,7 +348,7 @@ func (r *mavenResolver) cacheResolveReader(key string, resolve func() (io.ReadCl return bytes.NewBuffer(contents), err } -// resolveParent attempts to resolve +// resolveParent attempts to resolve the parent for the given pom func (r *mavenResolver) resolveParent(ctx context.Context, pom *gopom.Project) (*gopom.Project, error) { if pom == nil || pom.Parent == nil { return nil, nil @@ -351,10 +359,24 @@ func (r *mavenResolver) resolveParent(ctx context.Context, pom *gopom.Project) ( groupID := r.getPropertyValue(ctx, &pomWithoutParent, parent.GroupID) artifactID := r.getPropertyValue(ctx, &pomWithoutParent, parent.ArtifactID) version := r.getPropertyValue(ctx, &pomWithoutParent, parent.Version) + + // check cache before resolving + parentID := mavenID{groupID, artifactID, version} + if resolvedParent, ok := r.resolved[parentID]; ok { + return resolvedParent, nil + } + + // check if the pom exists in the fileResolver + parentPom := r.findParentPomByRelativePath(ctx, pom, parentID) + if parentPom != nil { + return parentPom, nil + } + + // find POM normally return r.findPom(ctx, groupID, artifactID, version) } -// Try to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies +// findInheritedVersion attempts to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies // //nolint:gocognit func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Project, pom *gopom.Project, groupID, artifactID string, resolving ...mavenID) (string, error) { @@ -371,7 +393,7 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr var version string // check for entries in dependencyManagement first - for _, dep := range directManagedDependencies(pom) { + for _, dep := range pomManagedDependencies(pom) { depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) if depGroupID == groupID && depArtifactID == artifactID { @@ -415,7 +437,7 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr } // check for inherited dependencies - for _, dep := range directDependencies(pom) { + for _, dep := range pomDependencies(pom) { depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) if depGroupID == groupID && depArtifactID == artifactID { @@ -448,7 +470,7 @@ func (r *mavenResolver) resolveLicenses(ctx context.Context, pom *gopom.Project, return nil, fmt.Errorf("maximum parent recursive depth (%v) reached: %v", r.cfg.MaxParentRecursiveDepth, processing) } - directLicenses := r.directLicenses(ctx, pom) + directLicenses := r.pomLicenses(ctx, pom) if len(directLicenses) > 0 { return directLicenses, nil } @@ -463,8 +485,8 @@ func (r *mavenResolver) resolveLicenses(ctx context.Context, pom *gopom.Project, return r.resolveLicenses(ctx, parent, append(processing, id)...) } -// directLicenses appends the directly specified licenses with non-empty name or url -func (r *mavenResolver) directLicenses(ctx context.Context, pom *gopom.Project) []gopom.License { +// pomLicenses appends the directly specified licenses with non-empty name or url +func (r *mavenResolver) pomLicenses(ctx context.Context, pom *gopom.Project) []gopom.License { var out []gopom.License for _, license := range deref(pom.Licenses) { // if we find non-empty licenses, return them @@ -477,8 +499,62 @@ func (r *mavenResolver) directLicenses(ctx context.Context, pom *gopom.Project) return out } -// directDependencies returns all direct dependencies in a project, including all defined in profiles -func directDependencies(pom *gopom.Project) []gopom.Dependency { +func (r *mavenResolver) findParentPomByRelativePath(ctx context.Context, pom *gopom.Project, parentID mavenID) *gopom.Project { + // don't resolve if no resolver + if r.fileResolver == nil { + return nil + } + + pomLocation, hasPomLocation := r.pomLocations[pom] + if !hasPomLocation || pom == nil || pom.Parent == nil { + return nil + } + relativePath := r.getPropertyValue(ctx, pom, pom.Parent.RelativePath) + if relativePath == "" { + return nil + } + p := pomLocation.Path() + p = path.Dir(p) + p = path.Join(p, relativePath) + p = path.Clean(p) + parentLocations, err := r.fileResolver.FilesByPath(p) + if err != nil || len(parentLocations) == 0 { + log.Debugf("parent not found in by relative path for: %v looking for: %v at %v err: %v", newMavenIDFromPom(pom), parentID, relativePath, err) + return nil + } + parentLocation := parentLocations[0] + + parentContents, err := r.fileResolver.FileContentsByLocation(parentLocation) + if err != nil || parentContents == nil { + log.Debugf("unable to get parent by relative path for: %v parent: %v at %v err: %v", newMavenIDFromPom(pom), parentID, parentLocation, err) + return nil + } + defer internal.CloseAndLogError(parentContents, parentLocation.RealPath) + parentPom, err := decodePomXML(parentContents) + if err != nil || parentPom == nil { + log.Debugf("unable to parse parent by relative path for: %v parent: %v at %v err: %v", newMavenIDFromPom(pom), parentID, parentLocation, err) + return nil + } + // ensure ids match + groupID := r.getPropertyValue(ctx, pom, parentPom.GroupID) + artifactID := r.getPropertyValue(ctx, pom, parentPom.ArtifactID) + version := r.getPropertyValue(ctx, pom, parentPom.Version) + + newParentID := mavenID{groupID, artifactID, version} + if newParentID != parentID { + log.Debugf("parent IDs do not match resolving parent by relative path for: %v parent: %v at %v, got: %v", newMavenIDFromPom(pom), parentID, parentLocation, newParentID) + return nil + } + + r.resolved[parentID] = parentPom + r.pomLocations[parentPom] = parentLocation // for any future parent relativepath lookups + + return parentPom +} + +// pomDependencies returns all dependencies directly defined in a project, including all defined in profiles. +// does not resolve parent dependencies +func pomDependencies(pom *gopom.Project) []gopom.Dependency { dependencies := deref(pom.Dependencies) for _, profile := range deref(pom.Profiles) { dependencies = append(dependencies, deref(profile.Dependencies)...) @@ -486,8 +562,9 @@ func directDependencies(pom *gopom.Project) []gopom.Dependency { return dependencies } -// directManagedDependencies returns all managed dependencies in a project, including all defined in profiles -func directManagedDependencies(pom *gopom.Project) []gopom.Dependency { +// pomManagedDependencies returns all directly defined managed dependencies in a project pom, including all defined in profiles. +// does not resolve parent managed dependencies +func pomManagedDependencies(pom *gopom.Project) []gopom.Dependency { var dependencies []gopom.Dependency if pom.DependencyManagement != nil { dependencies = append(dependencies, deref(pom.DependencyManagement.Dependencies)...) diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index 97d4e0888a8..575baed6c6c 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -2,6 +2,8 @@ package java import ( "context" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/internal/fileresolver" "net/http" "net/http/httptest" "os" @@ -155,7 +157,7 @@ func Test_resolveProperty(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - r := newMavenResolver(DefaultArchiveCatalogerConfig()) + r := newMavenResolver(nil, DefaultArchiveCatalogerConfig()) resolved := r.getPropertyValue(context.Background(), &test.pom, ptr(test.property)) require.Equal(t, test.expected, resolved) }) @@ -186,7 +188,7 @@ func Test_mavenResolverLocal(t *testing.T) { for _, test := range tests { t.Run(test.artifactID, func(t *testing.T) { ctx := context.Background() - r := newMavenResolver(ArchiveCatalogerConfig{ + r := newMavenResolver(nil, ArchiveCatalogerConfig{ UseNetwork: false, UseMavenLocalRepository: true, MavenLocalRepositoryDir: dir, @@ -227,7 +229,7 @@ func Test_mavenResolverRemote(t *testing.T) { for _, test := range tests { t.Run(test.artifactID, func(t *testing.T) { ctx := context.Background() - r := newMavenResolver(ArchiveCatalogerConfig{ + r := newMavenResolver(nil, ArchiveCatalogerConfig{ UseNetwork: true, UseMavenLocalRepository: false, MavenBaseURL: url, @@ -245,6 +247,38 @@ func Test_mavenResolverRemote(t *testing.T) { } } +func Test_relativePathParent(t *testing.T) { + resolver, err := fileresolver.NewFromDirectory("test-fixtures/pom/relative", "") + require.NoError(t, err) + + r := newMavenResolver(resolver, DefaultArchiveCatalogerConfig()) + locs, err := resolver.FilesByPath("child-1/pom.xml") + require.NoError(t, err) + require.Len(t, locs, 1) + + loc := locs[0] + contents, err := resolver.FileContentsByLocation(loc) + require.NoError(t, err) + defer internal.CloseAndLogError(contents, loc.RealPath) + + pom, err := decodePomXML(contents) + require.NoError(t, err) + + r.pomLocations[pom] = loc + + ctx := context.Background() + parent, err := r.resolveParent(ctx, pom) + require.NoError(t, err) + require.Contains(t, r.pomLocations, parent) + + parent, err = r.resolveParent(ctx, parent) + require.NoError(t, err) + require.Contains(t, r.pomLocations, parent) + + got := r.getPropertyValue(ctx, pom, ptr("${commons-exec_subversion}")) + require.Equal(t, "3", got) +} + // testRepo starts a remote maven repo serving all the pom files found in the given directory func testRepo(t *testing.T, dir string) (url string) { // mux is the HTTP request multiplexer used with the test server. diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 8154f4ed01f..27bee293d1f 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -22,16 +22,17 @@ import ( const pomXMLGlob = "*pom.xml" -func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, fileResolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) if err != nil || pom == nil { return nil, nil, err } - r := newMavenResolver(gap.cfg) + r := newMavenResolver(fileResolver, gap.cfg) + r.pomLocations[pom] = reader.Location // store the location this pom was resolved in order to attempt parent pom lookups var pkgs []pkg.Package - for _, dep := range directDependencies(pom) { + for _, dep := range pomDependencies(pom) { id := newMavenID(dep.GroupID, dep.ArtifactID, dep.Version) log.Tracef("adding dependency to SBOM: %v", id) p, err := newPackageFromDependency( diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index d2e355a380c..2806324ab5e 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -329,7 +329,7 @@ func Test_parsePomXMLProject(t *testing.T) { t.Run(test.name, func(t *testing.T) { fixture, err := os.Open(test.project.Path) assert.NoError(t, err) - r := newMavenResolver(ArchiveCatalogerConfig{}) + r := newMavenResolver(nil, ArchiveCatalogerConfig{}) pom, err := gopom.ParseFromReader(fixture) require.NoError(t, err) @@ -338,7 +338,7 @@ func Test_parsePomXMLProject(t *testing.T) { assert.NoError(t, err) assert.Equal(t, test.project, actual) - licenses := r.directLicenses(context.Background(), pom) + licenses := r.pomLicenses(context.Background(), pom) assert.NoError(t, err) assert.Equal(t, test.licenses, toPkgLicenses(&jarLocation, licenses)) }) @@ -399,7 +399,7 @@ func Test_pomParent(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - r := newMavenResolver(DefaultArchiveCatalogerConfig()) + r := newMavenResolver(nil, DefaultArchiveCatalogerConfig()) assert.Equal(t, test.expected, pomParent(context.Background(), &r, &gopom.Project{Parent: test.input})) }) } diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom index 2d42648e51b..6a72f2fd56d 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/child-one/1.3.6/child-one-1.3.6.pom @@ -18,7 +18,6 @@ my.org parent-one 3.11.0 - ../../parent-1/pom.xml child-one diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom index f4be4a02c43..4dd7d533f73 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-one/3.11.0/parent-one-3.11.0.pom @@ -6,7 +6,6 @@ my.org parent-two 13.7.8 - ../parent-2 my.org diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom index fd7b3ba6c3e..5864bfa6cf5 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom @@ -4,8 +4,8 @@ 4.0.0 my.org - parent-2 - 13 + parent-two + 13.7.8 pom diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/relative/child-1/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/relative/child-1/pom.xml new file mode 100644 index 00000000000..63ed7d474dd --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/relative/child-1/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + + + + + my.org + parent-one + 3.11.0 + ../parent-1/pom.xml + + + child-one + + ${project.one}.3.6 + jar + + + 3.12.0 + 4.2 + 4.12 + + + + + org.apache.commons + commons-lang3 + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml new file mode 100644 index 00000000000..69ff49eff0c --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + my.org + parent-two + 13.7.8 + ../parent-2/pom.xml + + + my.org + parent-one + 3.11.0 + pom + + + + 3.1${project.parent.version}.0 + 4.3 + + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-collections4 + ${commons.collections4.version} + + + junit + junit + ${commons.junit.version} + test + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-2/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-2/pom.xml new file mode 100644 index 00000000000..5864bfa6cf5 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-2/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + my.org + parent-two + 13.7.8 + pom + + + 3.14.0 + 4.4 + 1.12.0 + 4.13.2 + 3 + 1 + + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + org.apache.commons + commons-text + ${commons.text.version} + + + junit + junit + ${commons.junit.version} + test + + + + + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-collections4 + ${commons.collections4.version} + + + junit + junit + ${commons.junit.version} + test + + + + From 6472bdff59c85cd6ee61ff6cba7ffe16258cd4f1 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 23 Jul 2024 15:31:19 -0400 Subject: [PATCH 32/42] chore: lint-fix Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/maven_resolver_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index 575baed6c6c..3154b05ad51 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -2,8 +2,6 @@ package java import ( "context" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/internal/fileresolver" "net/http" "net/http/httptest" "os" @@ -13,6 +11,9 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/stretchr/testify/require" "github.com/vifraa/gopom" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/internal/fileresolver" ) func Test_resolveProperty(t *testing.T) { From a1fb9d7efd4e0d23500396adb909a2fedc353f1a Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 23 Jul 2024 18:14:54 -0400 Subject: [PATCH 33/42] chore: refactor pom cataloger to scan and index all poms in the resolver Signed-off-by: Keith Zantow --- .../internal/pkgtest/test_generic_parser.go | 5 + syft/pkg/cataloger/java/archive_parser.go | 7 +- .../pkg/cataloger/java/archive_parser_test.go | 9 +- syft/pkg/cataloger/java/cataloger.go | 7 +- syft/pkg/cataloger/java/maven_resolver.go | 4 +- .../pkg/cataloger/java/maven_resolver_test.go | 2 +- syft/pkg/cataloger/java/parse_pom_xml.go | 63 +- syft/pkg/cataloger/java/parse_pom_xml_test.go | 52 +- .../test-fixtures/pom/commons-text.pom.xml | 575 ------------------ .../pom/{relative => local}/child-1/pom.xml | 0 .../pom/local/commons-text-1.10.0/pom.xml | 263 ++++++++ .../example-java-app-maven}/pom.xml | 0 .../pom/{relative => local}/parent-1/pom.xml | 1 - .../pom/{relative => local}/parent-2/pom.xml | 0 14 files changed, 363 insertions(+), 625 deletions(-) delete mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/commons-text.pom.xml rename syft/pkg/cataloger/java/test-fixtures/pom/{relative => local}/child-1/pom.xml (100%) create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/local/commons-text-1.10.0/pom.xml rename syft/pkg/cataloger/java/test-fixtures/pom/{ => local/example-java-app-maven}/pom.xml (100%) rename syft/pkg/cataloger/java/test-fixtures/pom/{relative => local}/parent-1/pom.xml (98%) rename syft/pkg/cataloger/java/test-fixtures/pom/{relative => local}/parent-2/pom.xml (100%) diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index a3cf11dece8..2dcbb7f8bec 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -310,6 +310,11 @@ func TestFileParser(t *testing.T, fixturePath string, parser generic.Parser, exp NewCatalogTester().FromFile(t, fixturePath).Expects(expectedPkgs, expectedRelationships).TestParser(t, parser) } +func TestCataloger(t *testing.T, fixtureDir string, cataloger pkg.Cataloger, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) { + t.Helper() + NewCatalogTester().FromDirectory(t, fixtureDir).Expects(expectedPkgs, expectedRelationships).TestCataloger(t, cataloger) +} + func TestFileParserWithEnv(t *testing.T, fixturePath string, parser generic.Parser, env *generic.Environment, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) { t.Helper() diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index c6ed0377064..8f21702be69 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -10,6 +10,7 @@ import ( "github.com/vifraa/gopom" + "github.com/anchore/syft/internal" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -51,7 +52,7 @@ type archiveParser struct { fileInfo archiveFilename detectNested bool cfg ArchiveCatalogerConfig - maven mavenResolver + maven *mavenResolver } type genericArchiveParserAdapter struct { @@ -382,7 +383,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren parsedPom = proj } - pkgFromPom := newPackageFromMavenData(ctx, &j.maven, propertiesObj, parsedPom, parentPkg, j.location) + pkgFromPom := newPackageFromMavenData(ctx, j.maven, propertiesObj, parsedPom, parentPkg, j.location) if pkgFromPom != nil { pkgs = append(pkgs, *pkgFromPom) } @@ -396,7 +397,7 @@ func getDigestsFromArchive(archivePath string) ([]file.Digest, error) { if err != nil { return nil, fmt.Errorf("unable to open archive path (%s): %w", archivePath, err) } - defer archiveCloser.Close() + defer internal.CloseAndLogError(archiveCloser, archivePath) // grab and assign digest for the entire archive digests, err := intFile.NewDigestsFromFile(archiveCloser, javaArchiveHashes) diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 2a2f13569de..564ae4c3e1a 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/vifraa/gopom" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/license" @@ -54,13 +55,13 @@ func generateMockMavenHandler(responseFixture string) func(w http.ResponseWriter // Set the Content-Type header to indicate that the response is XML w.Header().Set("Content-Type", "application/xml") // Copy the file's content to the response writer - file, err := os.Open(responseFixture) + f, err := os.Open(responseFixture) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - defer file.Close() - _, err = io.Copy(w, file) + defer internal.CloseAndLogError(f, responseFixture) + _, err = io.Copy(w, f) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -1081,7 +1082,7 @@ func Test_newPackageFromMavenData(t *testing.T) { test.expectedParent.Locations = locations r := newMavenResolver(nil, DefaultArchiveCatalogerConfig()) - actualPackage := newPackageFromMavenData(context.Background(), &r, test.props, test.project, test.parent, file.NewLocation(virtualPath)) + actualPackage := newPackageFromMavenData(context.Background(), r, test.props, test.project, test.parent, file.NewLocation(virtualPath)) if test.expectedPackage == nil { require.Nil(t, actualPackage) } else { diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 35f4cbdb2e9..11e48b7f5ad 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -32,10 +32,9 @@ func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { // NewPomCataloger returns a cataloger capable of parsing dependencies from a pom.xml file. // Pom files list dependencies that maybe not be locally installed yet. func NewPomCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { - gap := newGenericArchiveParserAdapter(cfg) - - return generic.NewCataloger("java-pom-cataloger"). - WithParserByGlobs(gap.parsePomXML, "**/pom.xml") + return pomXMLCataloger{ + cfg: cfg, + } } // NewGradleLockfileCataloger returns a cataloger capable of parsing dependencies from a gradle.lockfile file. diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index a5f64478e36..47bfc369ee3 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -74,8 +74,8 @@ type mavenResolver struct { // newMavenResolver constructs a new mavenResolver with the given configuration. // NOTE: the fileResolver is optional and if provided will be used to resolve parent poms by relative path -func newMavenResolver(fileResolver file.Resolver, cfg ArchiveCatalogerConfig) mavenResolver { - return mavenResolver{ +func newMavenResolver(fileResolver file.Resolver, cfg ArchiveCatalogerConfig) *mavenResolver { + return &mavenResolver{ cfg: cfg, cache: cache.GetManager().GetCache("java/maven/repo", "v1"), resolved: map[mavenID]*gopom.Project{}, diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index 3154b05ad51..5a16a5b6e64 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -249,7 +249,7 @@ func Test_mavenResolverRemote(t *testing.T) { } func Test_relativePathParent(t *testing.T) { - resolver, err := fileresolver.NewFromDirectory("test-fixtures/pom/relative", "") + resolver, err := fileresolver.NewFromDirectory("test-fixtures/pom/local", "") require.NoError(t, err) r := newMavenResolver(resolver, DefaultArchiveCatalogerConfig()) diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 27bee293d1f..fa11d93672b 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -13,34 +13,79 @@ import ( "github.com/vifraa/gopom" "golang.org/x/net/html/charset" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/generic" ) const pomXMLGlob = "*pom.xml" -func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, fileResolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - pom, err := decodePomXML(reader) - if err != nil || pom == nil { +type pomXMLCataloger struct { + cfg ArchiveCatalogerConfig +} + +func (p pomXMLCataloger) Name() string { + return "java-pom-cataloger" +} + +func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + locations, err := fileResolver.FilesByGlob("**/pom.xml") + if err != nil { return nil, nil, err } - r := newMavenResolver(fileResolver, gap.cfg) - r.pomLocations[pom] = reader.Location // store the location this pom was resolved in order to attempt parent pom lookups + r := newMavenResolver(fileResolver, p.cfg) + + var poms []*gopom.Project + for _, pomLocation := range locations { + pom, err := readPomFromLocation(fileResolver, pomLocation) + if err != nil || pom == nil { + log.Debugf("error while getting contents for: %v %v", pomLocation.RealPath, err) + continue + } + + poms = append(poms, pom) + + // store information about this pom for future lookups + r.pomLocations[pom] = pomLocation + r.resolved[newMavenIDFromPom(pom)] = pom + } + + var pkgs []pkg.Package + for _, pom := range poms { + pkgs = append(pkgs, processPomXML(ctx, r, pom, r.pomLocations[pom])...) + } + return pkgs, nil, nil +} + +func readPomFromLocation(fileResolver file.Resolver, pomLocation file.Location) (*gopom.Project, error) { + contents, err := fileResolver.FileContentsByLocation(pomLocation) + if err != nil { + return nil, err + } + defer internal.CloseAndLogError(contents, pomLocation.RealPath) + + pom, err := decodePomXML(contents) + if err != nil || pom == nil { + return nil, err + } + return pom, nil +} +func processPomXML(ctx context.Context, r *mavenResolver, pom *gopom.Project, loc file.Location) []pkg.Package { var pkgs []pkg.Package + for _, dep := range pomDependencies(pom) { id := newMavenID(dep.GroupID, dep.ArtifactID, dep.Version) log.Tracef("adding dependency to SBOM: %v", id) p, err := newPackageFromDependency( ctx, - &r, + r, pom, dep, - reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), + loc.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) if err != nil { log.Debugf("error adding dependency %v: %v", id, err) @@ -51,7 +96,7 @@ func (gap genericArchiveParserAdapter) parsePomXML(ctx context.Context, fileReso pkgs = append(pkgs, *p) } - return pkgs, nil, nil + return pkgs } func newPomProject(ctx context.Context, r *mavenResolver, path string, pom *gopom.Project) *pkg.JavaPomProject { diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 2806324ab5e..201bbf2496c 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -21,11 +21,11 @@ import ( func Test_parsePomXML(t *testing.T) { tests := []struct { - input string + dir string expected []pkg.Package }{ { - input: "test-fixtures/pom/pom.xml", + dir: "test-fixtures/pom/local/example-java-app-maven", expected: []pkg.Package{ { Name: "joda-time", @@ -59,19 +59,19 @@ func Test_parsePomXML(t *testing.T) { } for _, test := range tests { - t.Run(test.input, func(t *testing.T) { + t.Run(test.dir, func(t *testing.T) { for i := range test.expected { - test.expected[i].Locations.Add(file.NewLocation(test.input)) + test.expected[i].Locations.Add(file.NewLocation("pom.xml")) } - gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + cat := NewPomCataloger(ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: true, }, }) - pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) + pkgtest.TestCataloger(t, test.dir, cat, test.expected, nil) }) } } @@ -132,30 +132,30 @@ func Test_decodePomXML_surviveNonUtf8Encoding(t *testing.T) { func Test_parseCommonsTextPomXMLProject(t *testing.T) { tests := []struct { - input string + dir string expected []pkg.Package }{ { - input: "test-fixtures/pom/commons-text.pom.xml", + dir: "test-fixtures/pom/local/commons-text-1.10.0", expected: getCommonsTextExpectedPackages(), }, } for _, test := range tests { - t.Run(test.input, func(t *testing.T) { + t.Run(test.dir, func(t *testing.T) { for i := range test.expected { - test.expected[i].Locations.Add(file.NewLocation(test.input)) + test.expected[i].Locations.Add(file.NewLocation("pom.xml")) } - gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + cat := NewPomCataloger(ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: true, }, UseMavenLocalRepository: false, }) - pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) + pkgtest.TestCataloger(t, test.dir, cat, test.expected, nil) }) } } @@ -180,22 +180,22 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { } tests := []struct { - input string + dir string expected []pkg.Package }{ { - input: "test-fixtures/pom/commons-text.pom.xml", + dir: "test-fixtures/pom/local/commons-text-1.10.0", expected: expectedPackages, }, } for _, test := range tests { - t.Run(test.input, func(t *testing.T) { + t.Run(test.dir, func(t *testing.T) { for i := range test.expected { - test.expected[i].Locations.Add(file.NewLocation(test.input)) + test.expected[i].Locations.Add(file.NewLocation("pom.xml")) } - gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + cat := NewPomCataloger(ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: true, @@ -204,7 +204,7 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { MavenLocalRepositoryDir: "test-fixtures/pom/maven-repo", MaxParentRecursiveDepth: 5, }) - pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) + pkgtest.TestCataloger(t, test.dir, cat, test.expected, nil) }) } } @@ -231,22 +231,22 @@ func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { } tests := []struct { - input string + dir string expected []pkg.Package }{ { - input: "test-fixtures/pom/commons-text.pom.xml", + dir: "test-fixtures/pom/local/commons-text-1.10.0", expected: expectedPackages, }, } for _, test := range tests { - t.Run(test.input, func(t *testing.T) { + t.Run(test.dir, func(t *testing.T) { for i := range test.expected { - test.expected[i].Locations.Add(file.NewLocation(test.input)) + test.expected[i].Locations.Add(file.NewLocation("pom.xml")) } - gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + cat := NewPomCataloger(ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: true, @@ -256,7 +256,7 @@ func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { UseMavenLocalRepository: false, MaxParentRecursiveDepth: 5, }) - pkgtest.TestFileParser(t, test.input, gap.parsePomXML, test.expected, nil) + pkgtest.TestCataloger(t, test.dir, cat, test.expected, nil) }) } } @@ -334,7 +334,7 @@ func Test_parsePomXMLProject(t *testing.T) { pom, err := gopom.ParseFromReader(fixture) require.NoError(t, err) - actual := newPomProject(context.Background(), &r, fixture.Name(), pom) + actual := newPomProject(context.Background(), r, fixture.Name(), pom) assert.NoError(t, err) assert.Equal(t, test.project, actual) @@ -400,7 +400,7 @@ func Test_pomParent(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := newMavenResolver(nil, DefaultArchiveCatalogerConfig()) - assert.Equal(t, test.expected, pomParent(context.Background(), &r, &gopom.Project{Parent: test.input})) + assert.Equal(t, test.expected, pomParent(context.Background(), r, &gopom.Project{Parent: test.input})) }) } } diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/commons-text.pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/commons-text.pom.xml deleted file mode 100644 index 6f54a6ed6b1..00000000000 --- a/syft/pkg/cataloger/java/test-fixtures/pom/commons-text.pom.xml +++ /dev/null @@ -1,575 +0,0 @@ - - - - 4.0.0 - - org.apache.commons - commons-parent - 54 - - commons-text - 1.10.0 - Apache Commons Text - Apache Commons Text is a library focused on algorithms working on strings. - https://commons.apache.org/proper/commons-text - - - ISO-8859-1 - UTF-8 - 1.8 - 1.8 - - text - org.apache.commons.text - - 1.10.0 - (Java 8+) - - TEXT - 12318221 - - text - https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-text - site-content - - 5.9.1 - 3.2.0 - 9.3 - - 4.7.2.0 - 4.7.2 - 3.19.0 - 6.49.0 - - 4.8.0 - 0.8.8 - - - 3.10.0 - 3.4.1 - - - 22.0.0.2 - 1.4 - - 0.16.0 - false - - 1.35 - 3.1.2 - - - 1.9 - RC1 - true - scm:svn:https://dist.apache.org/repos/dist/dev/commons/${commons.componentid} - Gary Gregory - 86fdc7e2a11262cb - - - - - org.apache.commons - commons-lang3 - 3.12.0 - - - - org.junit.jupiter - junit-jupiter - test - - - org.assertj - assertj-core - 3.23.1 - test - - - commons-io - commons-io - 2.11.0 - test - - - org.mockito - - mockito-inline - ${commons.mockito.version} - test - - - org.graalvm.js - js - ${graalvm.version} - test - - - org.graalvm.js - js-scriptengine - ${graalvm.version} - test - - - org.apache.commons - commons-rng-simple - ${commons.rng.version} - test - - - org.openjdk.jmh - jmh-core - ${jmh.version} - test - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - test - - - - - clean verify apache-rat:check japicmp:cmp checkstyle:check spotbugs:check javadoc:javadoc - - - - org.apache.rat - apache-rat-plugin - - - site-content/** - src/site/resources/download_lang.cgi - src/test/resources/org/apache/commons/text/stringEscapeUtilsTestData.txt - src/test/resources/org/apache/commons/text/lcs-perf-analysis-inputs.csv - src/site/resources/release-notes/RELEASE-NOTES-*.txt - - - - - maven-pmd-plugin - ${commons.pmd.version} - - ${maven.compiler.target} - - - - net.sourceforge.pmd - pmd-core - ${commons.pmd-impl.version} - - - net.sourceforge.pmd - pmd-java - ${commons.pmd-impl.version} - - - net.sourceforge.pmd - pmd-javascript - ${commons.pmd-impl.version} - - - net.sourceforge.pmd - pmd-jsp - ${commons.pmd-impl.version} - - - - - - - - maven-checkstyle-plugin - ${checkstyle.plugin.version} - - false - src/conf/checkstyle.xml - src/conf/checkstyle-header.txt - src/conf/checkstyle-suppressions.xml - src/conf/checkstyle-suppressions.xml - true - **/generated/**.java,**/jmh_generated/**.java - - - - com.puppycrawl.tools - checkstyle - ${checkstyle.version} - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${commons.spotbugs.plugin.version} - - - com.github.spotbugs - spotbugs - ${commons.spotbugs.impl.version} - - - - src/conf/spotbugs-exclude-filter.xml - - - - maven-assembly-plugin - - - src/assembly/bin.xml - src/assembly/src.xml - - gnu - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - - ${commons.module.name} - - - - - - org.apache.maven.plugins - maven-scm-publish-plugin - - - javadocs - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - ${maven.compiler.source} - - - - - - - - - maven-checkstyle-plugin - ${checkstyle.plugin.version} - - false - src/conf/checkstyle.xml - src/conf/checkstyle-header.txt - src/conf/checkstyle-suppressions.xml - src/conf/checkstyle-suppressions.xml - true - **/generated/**.java,**/jmh_generated/**.java - - - - - checkstyle - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${commons.spotbugs.plugin.version} - - src/conf/spotbugs-exclude-filter.xml - - - - com.github.siom79.japicmp - japicmp-maven-plugin - - - maven-pmd-plugin - 3.19.0 - - ${maven.compiler.target} - - - - - pmd - cpd - - - - - - org.codehaus.mojo - taglist-maven-plugin - 3.0.0 - - - - - Needs Work - - - TODO - exact - - - FIXME - exact - - - XXX - exact - - - - - Noteable Markers - - - NOTE - exact - - - NOPMD - exact - - - NOSONAR - exact - - - - - - - - - - - 2014 - - - - kinow - Bruno P. Kinoshita - kinow@apache.org - - - britter - Benedikt Ritter - britter@apache.org - - - chtompki - Rob Tompkins - chtompki@apache.org - - - ggregory - Gary Gregory - ggregory at apache.org - https://www.garygregory.com - The Apache Software Foundation - https://www.apache.org/ - - PMC Member - - America/New_York - - https://people.apache.org/~ggregory/img/garydgregory80.png - - - - djones - Duncan Jones - djones@apache.org - - - - - - Don Jeba - donjeba@yahoo.com - - - Sampanna Kahu - - - Jarek Strzelecki - - - Lee Adcock - - - Amey Jadiye - ameyjadiye@gmail.com - - - Arun Vinud S S - - - Ioannis Sermetziadis - - - Jostein Tveit - - - Luciano Medallia - - - Jan Martin Keil - - - Nandor Kollar - - - Nick Wong - - - Ali Ghanbari - https://ali-ghanbari.github.io/ - - - - - scm:git:https://gitbox.apache.org/repos/asf/commons-text - scm:git:https://gitbox.apache.org/repos/asf/commons-text - https://gitbox.apache.org/repos/asf?p=commons-text.git - - - - jira - https://issues.apache.org/jira/browse/TEXT - - - - - apache.website - Apache Commons Site - scm:svn:https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-text/ - - - - - - setup-checkout - - - site-content - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - prepare-checkout - - run - - pre-site - - - - - - - - - - - - - - - - - - - - - - - - java9+ - - [9,) - - - - true - - - - benchmark - - true - org.apache - - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - benchmark - test - - exec - - - test - java - - -classpath - - org.openjdk.jmh.Main - -rf - json - -rff - target/jmh-result.${benchmark}.json - ${benchmark} - - - - - - - - - - \ No newline at end of file diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/relative/child-1/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/child-1/pom.xml similarity index 100% rename from syft/pkg/cataloger/java/test-fixtures/pom/relative/child-1/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/pom/local/child-1/pom.xml diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/local/commons-text-1.10.0/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/commons-text-1.10.0/pom.xml new file mode 100644 index 00000000000..e4ad83f1596 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/local/commons-text-1.10.0/pom.xml @@ -0,0 +1,263 @@ + + + + 4.0.0 + + org.apache.commons + commons-parent + 54 + + commons-text + 1.10.0 + Apache Commons Text + Apache Commons Text is a library focused on algorithms working on strings. + https://commons.apache.org/proper/commons-text + + + ISO-8859-1 + UTF-8 + 1.8 + 1.8 + + text + org.apache.commons.text + + 1.10.0 + (Java 8+) + + TEXT + 12318221 + + text + https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-text + site-content + + 5.9.1 + 3.2.0 + 9.3 + + 4.7.2.0 + 4.7.2 + 3.19.0 + 6.49.0 + + 4.8.0 + 0.8.8 + + + 3.10.0 + 3.4.1 + + + 22.0.0.2 + 1.4 + + 0.16.0 + false + + 1.35 + 3.1.2 + + + 1.9 + RC1 + true + scm:svn:https://dist.apache.org/repos/dist/dev/commons/${commons.componentid} + Gary Gregory + 86fdc7e2a11262cb + + + + + org.apache.commons + commons-lang3 + 3.12.0 + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + 3.23.1 + test + + + commons-io + commons-io + 2.11.0 + test + + + org.mockito + + mockito-inline + ${commons.mockito.version} + test + + + org.graalvm.js + js + ${graalvm.version} + test + + + org.graalvm.js + js-scriptengine + ${graalvm.version} + test + + + org.apache.commons + commons-rng-simple + ${commons.rng.version} + test + + + org.openjdk.jmh + jmh-core + ${jmh.version} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + test + + + + 2014 + + + scm:git:https://gitbox.apache.org/repos/asf/commons-text + scm:git:https://gitbox.apache.org/repos/asf/commons-text + https://gitbox.apache.org/repos/asf?p=commons-text.git + + + + jira + https://issues.apache.org/jira/browse/TEXT + + + + + apache.website + Apache Commons Site + scm:svn:https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-text/ + + + + + + setup-checkout + + + site-content + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + prepare-checkout + + run + + pre-site + + + + + + + + + + + + + + + + + + + + + + + + java9+ + + [9,) + + + + true + + + + benchmark + + true + org.apache + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + benchmark + test + + exec + + + test + java + + -classpath + + org.openjdk.jmh.Main + -rf + json + -rff + target/jmh-result.${benchmark}.json + ${benchmark} + + + + + + + + + + \ No newline at end of file diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/example-java-app-maven/pom.xml similarity index 100% rename from syft/pkg/cataloger/java/test-fixtures/pom/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/pom/local/example-java-app-maven/pom.xml diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-1/pom.xml similarity index 98% rename from syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/pom/local/parent-1/pom.xml index 69ff49eff0c..4a6d1f323c2 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-1/pom.xml +++ b/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-1/pom.xml @@ -9,7 +9,6 @@ ../parent-2/pom.xml - my.org parent-one 3.11.0 pom diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-2/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-2/pom.xml similarity index 100% rename from syft/pkg/cataloger/java/test-fixtures/pom/relative/parent-2/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/pom/local/parent-2/pom.xml From 7b2fb7a0484c31f5b112e845731536c3dcdc63c5 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 23 Jul 2024 19:12:39 -0400 Subject: [PATCH 34/42] chore: improve property resolution for boms Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/archive_parser.go | 6 +- syft/pkg/cataloger/java/maven_resolver.go | 102 +++++++++--------- .../pkg/cataloger/java/maven_resolver_test.go | 8 +- syft/pkg/cataloger/java/parse_pom_xml.go | 28 ++--- 4 files changed, 70 insertions(+), 74 deletions(-) diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 8f21702be69..33fbb7634c9 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -334,13 +334,13 @@ func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (gro if parsedPom != nil && parsedPom.project != nil { pom := parsedPom.project if group == "" { - group = j.maven.getPropertyValue(ctx, pom, pom.GroupID) + group = j.maven.getPropertyValue(ctx, pom.GroupID, pom) } if name == "" { - name = j.maven.getPropertyValue(ctx, pom, pom.ArtifactID) + name = j.maven.getPropertyValue(ctx, pom.ArtifactID, pom) } if version == "" { - version = j.maven.getPropertyValue(ctx, pom, pom.Version) + version = j.maven.getPropertyValue(ctx, pom.Version, pom) } } diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index 47bfc369ee3..c488b342d40 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -48,11 +48,6 @@ func newMavenIDFromPom(pom *gopom.Project) mavenID { return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) } -// Valid indicates this mavenID has non-empty groupId, artifactId, and version -func (m mavenID) Valid() bool { - return m.GroupID != "" && m.ArtifactID != "" && m.Version != "" -} - func (m mavenID) String() string { return fmt.Sprintf("(groupId: %s artifactId: %s version: %s)", m.GroupID, m.ArtifactID, m.Version) } @@ -90,11 +85,11 @@ func newMavenResolver(fileResolver file.Resolver, cfg ArchiveCatalogerConfig) *m // Properties which are not resolved result in empty string "" // //nolint:gocognit -func (r *mavenResolver) getPropertyValue(ctx context.Context, pom *gopom.Project, propertyValue *string) string { +func (r *mavenResolver) getPropertyValue(ctx context.Context, propertyValue *string, resolutionContext ...*gopom.Project) string { if propertyValue == nil { return "" } - resolved, err := r.resolveExpression(ctx, pom, *propertyValue, nil) + resolved, err := r.resolveExpression(ctx, resolutionContext, *propertyValue, nil) if err != nil { log.Debugf("error resolving maven property: %s: %v", *propertyValue, err) return "" @@ -105,11 +100,11 @@ func (r *mavenResolver) getPropertyValue(ctx context.Context, pom *gopom.Project // resolveExpression resolves an expression, which may be a plain string or a string with ${ property.references } // //nolint:gocognit -func (r *mavenResolver) resolveExpression(ctx context.Context, pom *gopom.Project, expression string, resolving []string) (string, error) { +func (r *mavenResolver) resolveExpression(ctx context.Context, resolutionContext []*gopom.Project, expression string, resolving []string) (string, error) { var err error return expressionMatcher.ReplaceAllStringFunc(expression, func(match string) string { propertyExpression := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } - resolved, e := r.resolveProperty(ctx, pom, propertyExpression, resolving) + resolved, e := r.resolveProperty(ctx, resolutionContext, propertyExpression, resolving) if e != nil { err = errors.Join(err, e) return "" @@ -121,31 +116,33 @@ func (r *mavenResolver) resolveExpression(ctx context.Context, pom *gopom.Projec // resolveProperty resolves properties recursively from the root project // //nolint:gocognit -func (r *mavenResolver) resolveProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { +func (r *mavenResolver) resolveProperty(ctx context.Context, resolutionContext []*gopom.Project, propertyExpression string, resolving []string) (string, error) { // prevent cycles if slices.Contains(resolving, propertyExpression) { return "", fmt.Errorf("cycle detected resolving: %s", propertyExpression) } resolving = append(resolving, propertyExpression) - value, err := r.resolveProjectProperty(ctx, pom, propertyExpression, resolving) - if err != nil { - return value, err - } - if value != "" { - return value, nil - } + for _, pom := range resolutionContext { + value, err := r.resolveProjectProperty(ctx, resolutionContext, pom, propertyExpression, resolving) + if err != nil { + return value, err + } + if value != "" { + return value, nil + } - current := pom - for current != nil { - if current.Properties != nil && current.Properties.Entries != nil { - if value, ok := current.Properties.Entries[propertyExpression]; ok { - return r.resolveExpression(ctx, pom, value, resolving) // property values can contain expressions + current := pom + for current != nil { + if current.Properties != nil && current.Properties.Entries != nil { + if value, ok := current.Properties.Entries[propertyExpression]; ok { + return r.resolveExpression(ctx, resolutionContext, value, resolving) // property values can contain expressions + } + } + current, err = r.resolveParent(ctx, current) + if err != nil { + return "", err } - } - current, err = r.resolveParent(ctx, current) - if err != nil { - return "", err } } @@ -155,7 +152,7 @@ func (r *mavenResolver) resolveProperty(ctx context.Context, pom *gopom.Project, // resolveProjectProperty resolves properties on the project // //nolint:gocognit -func (r *mavenResolver) resolveProjectProperty(ctx context.Context, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { +func (r *mavenResolver) resolveProjectProperty(ctx context.Context, resolutionContext []*gopom.Project, pom *gopom.Project, propertyExpression string, resolving []string) (string, error) { // see if we have a project.x expression and process this based // on the xml tags in gopom parts := strings.Split(propertyExpression, ".") @@ -197,7 +194,7 @@ func (r *mavenResolver) resolveProjectProperty(ctx context.Context, pom *gopom.P // If this was the last part of the property name, return the value if partNum == numParts-1 { value := fmt.Sprintf("%v", pomValue.Interface()) - return r.resolveExpression(ctx, pom, value, resolving) + return r.resolveExpression(ctx, resolutionContext, value, resolving) } break } @@ -356,9 +353,9 @@ func (r *mavenResolver) resolveParent(ctx context.Context, pom *gopom.Project) ( parent := pom.Parent pomWithoutParent := *pom pomWithoutParent.Parent = nil - groupID := r.getPropertyValue(ctx, &pomWithoutParent, parent.GroupID) - artifactID := r.getPropertyValue(ctx, &pomWithoutParent, parent.ArtifactID) - version := r.getPropertyValue(ctx, &pomWithoutParent, parent.Version) + groupID := r.getPropertyValue(ctx, parent.GroupID, &pomWithoutParent) + artifactID := r.getPropertyValue(ctx, parent.ArtifactID, &pomWithoutParent) + version := r.getPropertyValue(ctx, parent.Version, &pomWithoutParent) // check cache before resolving parentID := mavenID{groupID, artifactID, version} @@ -379,25 +376,24 @@ func (r *mavenResolver) resolveParent(ctx context.Context, pom *gopom.Project) ( // findInheritedVersion attempts to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies // //nolint:gocognit -func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Project, pom *gopom.Project, groupID, artifactID string, resolving ...mavenID) (string, error) { - id := newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) - if len(resolving) >= r.cfg.MaxParentRecursiveDepth { - return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) +func (r *mavenResolver) findInheritedVersion(ctx context.Context, pom *gopom.Project, groupID, artifactID string, resolutionContext ...*gopom.Project) (string, error) { + if len(resolutionContext) >= r.cfg.MaxParentRecursiveDepth { + return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, newMavenIDFromPom(pom)) } - if slices.Contains(resolving, id) { - return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, resolving) + if slices.Contains(resolutionContext, pom) { + return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, newMavenIDFromPom(pom)) } - resolving = append(resolving, id) + resolutionContext = append(resolutionContext, pom) var err error var version string // check for entries in dependencyManagement first for _, dep := range pomManagedDependencies(pom) { - depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) - depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) + depGroupID := r.getPropertyValue(ctx, dep.GroupID, resolutionContext...) + depArtifactID := r.getPropertyValue(ctx, dep.ArtifactID, resolutionContext...) if depGroupID == groupID && depArtifactID == artifactID { - version = r.getPropertyValue(ctx, root, dep.Version) + version = r.getPropertyValue(ctx, dep.Version, resolutionContext...) if version != "" { return version, nil } @@ -405,13 +401,13 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr // imported pom files should be treated just like parent poms, they are used to define versions of dependencies if deref(dep.Type) == "pom" && deref(dep.Scope) == "import" { - depVersion := r.getPropertyValue(ctx, root, dep.Version) + depVersion := r.getPropertyValue(ctx, dep.Version, resolutionContext...) depPom, err := r.findPom(ctx, depGroupID, depArtifactID, depVersion) if err != nil { return "", err } - version, err = r.findInheritedVersion(ctx, root, depPom, groupID, artifactID, resolving...) + version, err = r.findInheritedVersion(ctx, depPom, groupID, artifactID, resolutionContext...) if err != nil { return "", err } @@ -427,7 +423,7 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr return "", err } if parent != nil { - version, err = r.findInheritedVersion(ctx, root, parent, groupID, artifactID, resolving...) + version, err = r.findInheritedVersion(ctx, parent, groupID, artifactID, resolutionContext...) if err != nil { return "", err } @@ -438,10 +434,10 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, root *gopom.Pr // check for inherited dependencies for _, dep := range pomDependencies(pom) { - depGroupID := r.getPropertyValue(ctx, root, dep.GroupID) - depArtifactID := r.getPropertyValue(ctx, root, dep.ArtifactID) + depGroupID := r.getPropertyValue(ctx, dep.GroupID, resolutionContext...) + depArtifactID := r.getPropertyValue(ctx, dep.ArtifactID, resolutionContext...) if depGroupID == groupID && depArtifactID == artifactID { - version = r.getPropertyValue(ctx, root, dep.Version) + version = r.getPropertyValue(ctx, dep.Version, resolutionContext...) if version != "" { return version, nil } @@ -490,8 +486,8 @@ func (r *mavenResolver) pomLicenses(ctx context.Context, pom *gopom.Project) []g var out []gopom.License for _, license := range deref(pom.Licenses) { // if we find non-empty licenses, return them - name := r.getPropertyValue(ctx, pom, license.Name) - url := r.getPropertyValue(ctx, pom, license.URL) + name := r.getPropertyValue(ctx, license.Name, pom) + url := r.getPropertyValue(ctx, license.URL, pom) if name != "" || url != "" { out = append(out, license) } @@ -509,7 +505,7 @@ func (r *mavenResolver) findParentPomByRelativePath(ctx context.Context, pom *go if !hasPomLocation || pom == nil || pom.Parent == nil { return nil } - relativePath := r.getPropertyValue(ctx, pom, pom.Parent.RelativePath) + relativePath := r.getPropertyValue(ctx, pom.Parent.RelativePath, pom) if relativePath == "" { return nil } @@ -536,9 +532,9 @@ func (r *mavenResolver) findParentPomByRelativePath(ctx context.Context, pom *go return nil } // ensure ids match - groupID := r.getPropertyValue(ctx, pom, parentPom.GroupID) - artifactID := r.getPropertyValue(ctx, pom, parentPom.ArtifactID) - version := r.getPropertyValue(ctx, pom, parentPom.Version) + groupID := r.getPropertyValue(ctx, parentPom.GroupID, pom) + artifactID := r.getPropertyValue(ctx, parentPom.ArtifactID, pom) + version := r.getPropertyValue(ctx, parentPom.Version, pom) newParentID := mavenID{groupID, artifactID, version} if newParentID != parentID { diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index 5a16a5b6e64..b3fe70a78b8 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -159,7 +159,7 @@ func Test_resolveProperty(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := newMavenResolver(nil, DefaultArchiveCatalogerConfig()) - resolved := r.getPropertyValue(context.Background(), &test.pom, ptr(test.property)) + resolved := r.getPropertyValue(context.Background(), ptr(test.property), &test.pom) require.Equal(t, test.expected, resolved) }) } @@ -201,7 +201,7 @@ func Test_mavenResolverLocal(t *testing.T) { } else { require.NoError(t, err) } - got := r.getPropertyValue(context.Background(), pom, &test.expression) + got := r.getPropertyValue(context.Background(), &test.expression, pom) require.Equal(t, test.expected, got) }) } @@ -242,7 +242,7 @@ func Test_mavenResolverRemote(t *testing.T) { } else { require.NoError(t, err) } - got := r.getPropertyValue(context.Background(), pom, &test.expression) + got := r.getPropertyValue(context.Background(), &test.expression, pom) require.Equal(t, test.expected, got) }) } @@ -276,7 +276,7 @@ func Test_relativePathParent(t *testing.T) { require.NoError(t, err) require.Contains(t, r.pomLocations, parent) - got := r.getPropertyValue(ctx, pom, ptr("${commons-exec_subversion}")) + got := r.getPropertyValue(ctx, ptr("${commons-exec_subversion}"), pom) require.Equal(t, "3", got) } diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index fa11d93672b..c9b3d1e34df 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -100,38 +100,38 @@ func processPomXML(ctx context.Context, r *mavenResolver, pom *gopom.Project, lo } func newPomProject(ctx context.Context, r *mavenResolver, path string, pom *gopom.Project) *pkg.JavaPomProject { - artifactID := r.getPropertyValue(ctx, pom, pom.ArtifactID) - name := r.getPropertyValue(ctx, pom, pom.Name) - projectURL := r.getPropertyValue(ctx, pom, pom.URL) + artifactID := r.getPropertyValue(ctx, pom.ArtifactID, pom) + name := r.getPropertyValue(ctx, pom.Name, pom) + projectURL := r.getPropertyValue(ctx, pom.URL, pom) log.WithFields("path", path, "artifactID", artifactID, "name", name, "projectURL", projectURL).Trace("parsing pom.xml") return &pkg.JavaPomProject{ Path: path, Parent: pomParent(ctx, r, pom), - GroupID: r.getPropertyValue(ctx, pom, pom.GroupID), + GroupID: r.getPropertyValue(ctx, pom.GroupID, pom), ArtifactID: artifactID, - Version: r.getPropertyValue(ctx, pom, pom.Version), + Version: r.getPropertyValue(ctx, pom.Version, pom), Name: name, - Description: cleanDescription(r.getPropertyValue(ctx, pom, pom.Description)), + Description: cleanDescription(r.getPropertyValue(ctx, pom.Description, pom)), URL: projectURL, } } func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom.Project, dep gopom.Dependency, locations ...file.Location) (*pkg.Package, error) { - groupID := r.getPropertyValue(ctx, pom, dep.GroupID) - artifactID := r.getPropertyValue(ctx, pom, dep.ArtifactID) - version := r.getPropertyValue(ctx, pom, dep.Version) + groupID := r.getPropertyValue(ctx, dep.GroupID, pom) + artifactID := r.getPropertyValue(ctx, dep.ArtifactID, pom) + version := r.getPropertyValue(ctx, dep.Version, pom) var err error if version == "" { - version, err = r.findInheritedVersion(ctx, pom, pom, groupID, artifactID) + version, err = r.findInheritedVersion(ctx, pom, groupID, artifactID) } m := pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: groupID, ArtifactID: artifactID, - Scope: r.getPropertyValue(ctx, pom, dep.Scope), + Scope: r.getPropertyValue(ctx, dep.Scope, pom), }, } @@ -224,9 +224,9 @@ func pomParent(ctx context.Context, r *mavenResolver, pom *gopom.Project) *pkg.J return nil } - groupID := r.getPropertyValue(ctx, pom, pom.Parent.GroupID) - artifactID := r.getPropertyValue(ctx, pom, pom.Parent.ArtifactID) - version := r.getPropertyValue(ctx, pom, pom.Parent.Version) + groupID := r.getPropertyValue(ctx, pom.Parent.GroupID, pom) + artifactID := r.getPropertyValue(ctx, pom.Parent.ArtifactID, pom) + version := r.getPropertyValue(ctx, pom.Parent.Version, pom) if groupID == "" && artifactID == "" && version == "" { return nil From 0f41319059f51532b7639a67ddcc78c40b76c9a4 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 24 Jul 2024 17:16:57 -0400 Subject: [PATCH 35/42] chore: properly resolve maven ID info Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/archive_parser.go | 20 +-- .../pkg/cataloger/java/archive_parser_test.go | 5 + syft/pkg/cataloger/java/maven_resolver.go | 100 ++++++++------- syft/pkg/cataloger/java/parse_pom_xml.go | 64 +++++----- syft/pkg/cataloger/java/parse_pom_xml_test.go | 115 +++++++++++++++++- .../pom/local/contains-child-1/pom.xml | 27 ++++ .../test-fixtures/pom/local/parent-2/pom.xml | 7 ++ .../parent-two/13.7.8/parent-two-13.7.8.pom | 7 ++ 8 files changed, 257 insertions(+), 88 deletions(-) create mode 100644 syft/pkg/cataloger/java/test-fixtures/pom/local/contains-child-1/pom.xml diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 33fbb7634c9..fc26ac95c69 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -233,9 +233,9 @@ func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest 3. manifest 4. filename */ - group, name, version, parsedPom := j.discoverMainPackageFromPomInfo(ctx) - if name == "" { - name = selectName(manifest, j.fileInfo) + groupID, artifactID, version, parsedPom := j.discoverMainPackageFromPomInfo(ctx) + if artifactID == "" { + artifactID = selectName(manifest, j.fileInfo) } if version == "" { version = selectVersion(manifest, j.fileInfo) @@ -256,10 +256,10 @@ func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest // Today we don't have a way to distinguish between licenses from the manifest and licenses from the pom.xml // until the file.Location object can support sub-paths (i.e. paths within archives, recursively; issue https://github.com/anchore/syft/issues/2211). // Until then it's less confusing to use the licenses from the pom.xml only if the manifest did not list any. - licenses = j.findLicenseFromJavaMetadata(ctx, group, name, version, parsedPom, manifest) + licenses = j.findLicenseFromJavaMetadata(ctx, groupID, artifactID, version, parsedPom, manifest) } - return name, version, licenses, nil + return artifactID, version, licenses, nil } // findLicenseFromJavaMetadata attempts to find license information from all available maven metadata properties and pom info @@ -275,7 +275,7 @@ func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID if parsedPom != nil { pomLicenses, err = j.maven.resolveLicenses(ctx, parsedPom.project) if err != nil { - log.Debugf("error attempting to resolve licenses for %v: %v", newMavenIDFromPom(parsedPom.project), err) + log.Debugf("error attempting to resolve licenses for %v: %v", j.maven.resolveMavenID(ctx, parsedPom.project), err) } } @@ -332,15 +332,15 @@ func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (gro version = pomProperties.Version if parsedPom != nil && parsedPom.project != nil { - pom := parsedPom.project + id := j.maven.resolveMavenID(ctx, parsedPom.project) if group == "" { - group = j.maven.getPropertyValue(ctx, pom.GroupID, pom) + group = id.GroupID } if name == "" { - name = j.maven.getPropertyValue(ctx, pom.ArtifactID, pom) + name = id.ArtifactID } if version == "" { - version = j.maven.getPropertyValue(ctx, pom.Version, pom) + version = id.Version } } diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 564ae4c3e1a..8b2caf130ca 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -1427,3 +1427,8 @@ func run(t testing.TB, cmd *exec.Cmd) { } } } + +// ptr returns a pointer to the given value +func ptr[T any](value T) *T { + return &value +} diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index c488b342d40..6265dfc3032 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -31,21 +31,26 @@ type mavenID struct { Version string } -// newMavenID is a convenience to construct a mavenId from string pointers, used by gopom -func newMavenID(groupID, artifactID, version *string) mavenID { - return mavenID{ - GroupID: deref(groupID), - ArtifactID: deref(artifactID), - Version: deref(version), - } -} - -// newMavenIDFromPom creates a new mavenID from a pom -func newMavenIDFromPom(pom *gopom.Project) mavenID { +// resolveMavenID creates a new mavenID from a pom, resolving parent information as necessary +func (r *mavenResolver) resolveMavenID(ctx context.Context, pom *gopom.Project) mavenID { if pom == nil { return mavenID{} } - return newMavenID(pom.GroupID, pom.ArtifactID, pom.Version) + groupID := r.getPropertyValue(ctx, pom.GroupID, pom) + artifactID := r.getPropertyValue(ctx, pom.ArtifactID, pom) + version := r.getPropertyValue(ctx, pom.Version, pom) + if pom.Parent != nil { + if groupID == "" { + groupID = r.getPropertyValue(ctx, pom.Parent.GroupID, pom) + } + if artifactID == "" { + artifactID = r.getPropertyValue(ctx, pom.Parent.ArtifactID, pom) + } + if version == "" { + version = r.getPropertyValue(ctx, pom.Parent.Version, pom) + } + } + return mavenID{groupID, artifactID, version} } func (m mavenID) String() string { @@ -121,17 +126,21 @@ func (r *mavenResolver) resolveProperty(ctx context.Context, resolutionContext [ if slices.Contains(resolving, propertyExpression) { return "", fmt.Errorf("cycle detected resolving: %s", propertyExpression) } + if len(resolutionContext) == 0 { + return "", fmt.Errorf("no project variable resolution context provided for expression: '%s'", propertyExpression) + } resolving = append(resolving, propertyExpression) - for _, pom := range resolutionContext { - value, err := r.resolveProjectProperty(ctx, resolutionContext, pom, propertyExpression, resolving) - if err != nil { - return value, err - } - if value != "" { - return value, nil - } + // only resolve project. properties in the context of the current project pom + value, err := r.resolveProjectProperty(ctx, resolutionContext, resolutionContext[len(resolutionContext)-1], propertyExpression, resolving) + if err != nil { + return value, err + } + if value != "" { + return value, nil + } + for _, pom := range resolutionContext { current := pom for current != nil { if current.Properties != nil && current.Properties.Entries != nil { @@ -166,6 +175,19 @@ func (r *mavenResolver) resolveProjectProperty(ctx context.Context, resolutionCo } part := parts[partNum] + // these two fields are directly inherited from the pom parent values + if partNum == 1 && pom.Parent != nil { + switch part { + case "version": + if pom.Version == nil && pom.Parent.Version != nil { + return r.resolveExpression(ctx, resolutionContext, *pom.Parent.Version, resolving) + } + case "groupID": + if pom.GroupID == nil && pom.Parent.GroupID != nil { + return r.resolveExpression(ctx, resolutionContext, *pom.Parent.GroupID, resolving) + } + } + } for fieldNum := 0; fieldNum < pomValueType.NumField(); fieldNum++ { f := pomValueType.Field(fieldNum) tag := f.Tag.Get("xml") @@ -238,11 +260,7 @@ func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, versio errs = errors.Join(errs, err) } - if errs != nil { - return nil, fmt.Errorf("unable to resolve pom %s %s %s: %w", groupID, artifactID, version, errs) - } - - return nil, nil + return nil, fmt.Errorf("unable to resolve pom %s %s %s: %w", groupID, artifactID, version, errs) } // findPomInLocalRepository attempts to get the POM from the users local maven repository @@ -375,13 +393,16 @@ func (r *mavenResolver) resolveParent(ctx context.Context, pom *gopom.Project) ( // findInheritedVersion attempts to find the version of a dependency (groupID, artifactID) by searching all parent poms and imported managed dependencies // -//nolint:gocognit +//nolint:gocognit,funlen func (r *mavenResolver) findInheritedVersion(ctx context.Context, pom *gopom.Project, groupID, artifactID string, resolutionContext ...*gopom.Project) (string, error) { + if pom == nil { + return "", fmt.Errorf("nil pom provided to findInheritedVersion") + } if len(resolutionContext) >= r.cfg.MaxParentRecursiveDepth { - return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, newMavenIDFromPom(pom)) + return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, r.resolveMavenID(ctx, pom)) } if slices.Contains(resolutionContext, pom) { - return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, newMavenIDFromPom(pom)) + return "", fmt.Errorf("cycle detected attempting to resolve version for: %s:%s at: %v", groupID, artifactID, r.resolveMavenID(ctx, pom)) } resolutionContext = append(resolutionContext, pom) @@ -404,12 +425,13 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, pom *gopom.Pro depVersion := r.getPropertyValue(ctx, dep.Version, resolutionContext...) depPom, err := r.findPom(ctx, depGroupID, depArtifactID, depVersion) - if err != nil { - return "", err + if err != nil || depPom == nil { + log.Debugf("unable to find imported pom looking for managed dependencies for: %v for dependency: %v: %v", r.resolveMavenID(ctx, pom), mavenID{depGroupID, depArtifactID, depVersion}, err) + continue } version, err = r.findInheritedVersion(ctx, depPom, groupID, artifactID, resolutionContext...) if err != nil { - return "", err + log.Debugf("error calling findInheritedVersion for: %v for dependency: %v: %v", r.resolveMavenID(ctx, pom), mavenID{depGroupID, depArtifactID, depVersion}, err) } if version != "" { return version, nil @@ -458,7 +480,7 @@ func (r *mavenResolver) findLicenses(ctx context.Context, groupID, artifactID, v // resolveLicenses searches the pom for license, traversing parent poms if needed func (r *mavenResolver) resolveLicenses(ctx context.Context, pom *gopom.Project, processing ...mavenID) ([]gopom.License, error) { - id := newMavenIDFromPom(pom) + id := r.resolveMavenID(ctx, pom) if slices.Contains(processing, id) { return nil, fmt.Errorf("cycle detected resolving licenses for: %v", id) } @@ -515,30 +537,26 @@ func (r *mavenResolver) findParentPomByRelativePath(ctx context.Context, pom *go p = path.Clean(p) parentLocations, err := r.fileResolver.FilesByPath(p) if err != nil || len(parentLocations) == 0 { - log.Debugf("parent not found in by relative path for: %v looking for: %v at %v err: %v", newMavenIDFromPom(pom), parentID, relativePath, err) + log.Debugf("parent not found in by relative path for: %v looking for: %v at %v err: %v", r.resolveMavenID(ctx, pom), parentID, relativePath, err) return nil } parentLocation := parentLocations[0] parentContents, err := r.fileResolver.FileContentsByLocation(parentLocation) if err != nil || parentContents == nil { - log.Debugf("unable to get parent by relative path for: %v parent: %v at %v err: %v", newMavenIDFromPom(pom), parentID, parentLocation, err) + log.Debugf("unable to get parent by relative path for: %v parent: %v at %v err: %v", r.resolveMavenID(ctx, pom), parentID, parentLocation, err) return nil } defer internal.CloseAndLogError(parentContents, parentLocation.RealPath) parentPom, err := decodePomXML(parentContents) if err != nil || parentPom == nil { - log.Debugf("unable to parse parent by relative path for: %v parent: %v at %v err: %v", newMavenIDFromPom(pom), parentID, parentLocation, err) + log.Debugf("unable to parse parent by relative path for: %v parent: %v at %v err: %v", r.resolveMavenID(ctx, pom), parentID, parentLocation, err) return nil } // ensure ids match - groupID := r.getPropertyValue(ctx, parentPom.GroupID, pom) - artifactID := r.getPropertyValue(ctx, parentPom.ArtifactID, pom) - version := r.getPropertyValue(ctx, parentPom.Version, pom) - - newParentID := mavenID{groupID, artifactID, version} + newParentID := r.resolveMavenID(ctx, parentPom) if newParentID != parentID { - log.Debugf("parent IDs do not match resolving parent by relative path for: %v parent: %v at %v, got: %v", newMavenIDFromPom(pom), parentID, parentLocation, newParentID) + log.Debugf("parent IDs do not match resolving parent by relative path for: %v parent: %v at %v, got: %v", r.resolveMavenID(ctx, pom), parentID, parentLocation, newParentID) return nil } diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index c9b3d1e34df..b9df0ee7001 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -20,14 +20,17 @@ import ( "github.com/anchore/syft/syft/pkg" ) -const pomXMLGlob = "*pom.xml" +const ( + pomXMLGlob = "*pom.xml" + pomCatalogerName = "java-pom-cataloger" +) type pomXMLCataloger struct { cfg ArchiveCatalogerConfig } func (p pomXMLCataloger) Name() string { - return "java-pom-cataloger" + return pomCatalogerName } func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { @@ -50,7 +53,7 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver // store information about this pom for future lookups r.pomLocations[pom] = pomLocation - r.resolved[newMavenIDFromPom(pom)] = pom + r.resolved[r.resolveMavenID(ctx, pom)] = pom } var pkgs []pkg.Package @@ -66,20 +69,19 @@ func readPomFromLocation(fileResolver file.Resolver, pomLocation file.Location) return nil, err } defer internal.CloseAndLogError(contents, pomLocation.RealPath) - - pom, err := decodePomXML(contents) - if err != nil || pom == nil { - return nil, err - } - return pom, nil + return decodePomXML(contents) } func processPomXML(ctx context.Context, r *mavenResolver, pom *gopom.Project, loc file.Location) []pkg.Package { var pkgs []pkg.Package for _, dep := range pomDependencies(pom) { - id := newMavenID(dep.GroupID, dep.ArtifactID, dep.Version) - log.Tracef("adding dependency to SBOM: %v", id) + id := mavenID{ + r.getPropertyValue(ctx, dep.GroupID, pom), + r.getPropertyValue(ctx, dep.ArtifactID, pom), + r.getPropertyValue(ctx, dep.Version, pom), + } + log.Tracef("adding dependency to SBOM: %v from: %v", id, r.resolveMavenID(ctx, pom)) p, err := newPackageFromDependency( ctx, r, @@ -100,17 +102,17 @@ func processPomXML(ctx context.Context, r *mavenResolver, pom *gopom.Project, lo } func newPomProject(ctx context.Context, r *mavenResolver, path string, pom *gopom.Project) *pkg.JavaPomProject { - artifactID := r.getPropertyValue(ctx, pom.ArtifactID, pom) + id := r.resolveMavenID(ctx, pom) name := r.getPropertyValue(ctx, pom.Name, pom) projectURL := r.getPropertyValue(ctx, pom.URL, pom) - log.WithFields("path", path, "artifactID", artifactID, "name", name, "projectURL", projectURL).Trace("parsing pom.xml") + log.WithFields("path", path, "artifactID", id.ArtifactID, "name", name, "projectURL", projectURL).Trace("parsing pom.xml") return &pkg.JavaPomProject{ Path: path, Parent: pomParent(ctx, r, pom), - GroupID: r.getPropertyValue(ctx, pom.GroupID, pom), - ArtifactID: artifactID, - Version: r.getPropertyValue(ctx, pom.Version, pom), + GroupID: id.GroupID, + ArtifactID: id.ArtifactID, + Version: id.Version, Name: name, Description: cleanDescription(r.getPropertyValue(ctx, pom.Description, pom)), URL: projectURL, @@ -136,17 +138,15 @@ func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom. } var licenses []pkg.License - if version == "" { - dependencyPom, depErr := r.findPom(ctx, groupID, artifactID, version) - if depErr != nil { - log.Debugf("error getting licenses for %s: %v", mavenID{groupID, artifactID, version}, err) - err = errors.Join(err, depErr) - } - if dependencyPom != nil { - depLicenses, _ := r.resolveLicenses(ctx, dependencyPom) - for _, license := range depLicenses { - licenses = append(licenses, pkg.NewLicenseFromFields(deref(license.Name), deref(license.URL), nil)) - } + dependencyPom, depErr := r.findPom(ctx, groupID, artifactID, version) + if depErr != nil { + log.Debugf("error getting licenses for %s: %v", mavenID{groupID, artifactID, version}, err) + err = errors.Join(err, depErr) + } + if dependencyPom != nil { + depLicenses, _ := r.resolveLicenses(ctx, dependencyPom) + for _, license := range depLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(deref(license.Name), deref(license.URL), nil)) } } @@ -158,6 +158,7 @@ func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom. PURL: packageURL(artifactID, version, m), Language: pkg.Java, Type: pkg.JavaPkg, // TODO: should we differentiate between packages from jar/war/zip versus packages from a pom.xml that were not installed yet? + FoundBy: pomCatalogerName, Metadata: m, } @@ -166,6 +167,7 @@ func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom. return p, err } +// decodePomXML decodes a pom XML file, detecting and converting non-UTF-8 charsets. this DOES NOT perform any logic to resolve properties such as groupID, artifactID, and version func decodePomXML(content io.Reader) (project *gopom.Project, err error) { inputReader, err := getUtf8Reader(content) if err != nil { @@ -181,14 +183,6 @@ func decodePomXML(content io.Reader) (project *gopom.Project, err error) { return nil, fmt.Errorf("unable to unmarshal pom.xml: %w", err) } - // For modules groupID and version are almost always inherited from parent pom - if project.GroupID == nil && project.Parent != nil { - project.GroupID = project.Parent.GroupID - } - if project.Version == nil && project.Parent != nil { - project.Version = project.Parent.Version - } - return project, nil } diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 201bbf2496c..3068e2a1f6e 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -17,6 +17,8 @@ import ( "github.com/anchore/syft/syft/license" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" ) func Test_parsePomXML(t *testing.T) { @@ -33,6 +35,7 @@ func Test_parsePomXML(t *testing.T) { PURL: "pkg:maven/com.joda/joda-time@2.9.2", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "com.joda", @@ -46,6 +49,7 @@ func Test_parsePomXML(t *testing.T) { PURL: "pkg:maven/junit/junit@4.12", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "junit", @@ -428,8 +432,105 @@ func Test_cleanDescription(t *testing.T) { } } -func ptr[T any](value T) *T { - return &value +func Test_resolveLicenses(t *testing.T) { + mavenURL := testRepo(t, "test-fixtures/pom/maven-repo") + localM2 := "test-fixtures/pom/maven-repo" + localDir := "test-fixtures/pom/local" + containingDir := "test-fixtures/pom/local/contains-child-1" + + expectedLicenses := []pkg.License{ + { + Value: "Eclipse Public License v2.0", + SPDXExpression: "", + Type: license.Declared, + URLs: []string{"https://www.eclipse.org/legal/epl-v20.html"}, + }, + } + + tests := []struct { + name string + scanDir string + cfg ArchiveCatalogerConfig + expected []pkg.License + }{ + { + name: "local no resolution", + scanDir: containingDir, + cfg: ArchiveCatalogerConfig{ + UseMavenLocalRepository: false, + UseNetwork: false, + MavenLocalRepositoryDir: "", + MavenBaseURL: "", + MaxParentRecursiveDepth: 10, + }, + expected: nil, + }, + { + name: "local all poms", + scanDir: localDir, + cfg: ArchiveCatalogerConfig{ + UseMavenLocalRepository: false, + UseNetwork: false, + MaxParentRecursiveDepth: 10, + }, + expected: expectedLicenses, + }, + { + name: "local m2 cache", + scanDir: containingDir, + cfg: ArchiveCatalogerConfig{ + UseMavenLocalRepository: true, + MavenLocalRepositoryDir: localM2, + UseNetwork: false, + MavenBaseURL: "", + MaxParentRecursiveDepth: 10, + }, + expected: expectedLicenses, + }, + { + name: "local with network", + scanDir: containingDir, + cfg: ArchiveCatalogerConfig{ + UseMavenLocalRepository: false, + UseNetwork: true, + MavenBaseURL: mavenURL, + MaxParentRecursiveDepth: 10, + }, + expected: expectedLicenses, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cat := NewPomCataloger(test.cfg) + + ds, err := directorysource.NewFromPath(test.scanDir) + require.NoError(t, err) + + fr, err := ds.FileResolver(source.AllLayersScope) + require.NoError(t, err) + + ctx := context.TODO() + pkgs, _, err := cat.Catalog(ctx, fr) + require.NoError(t, err) + + var child1 pkg.Package + for _, p := range pkgs { + if p.Name == "child-one" { + child1 = p + break + } + } + require.Equal(t, "child-one", child1.Name) + + got := child1.Licenses.ToSlice() + for i := 0; i < len(got); i++ { + // ignore locations, just check license text + (&got[i]).Locations = file.LocationSet{} + } + require.ElementsMatch(t, test.expected, got) + }) + } } func Test_getUtf8Reader(t *testing.T) { @@ -465,6 +566,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.apache.commons/commons-lang3@3.12.0", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.apache.commons", @@ -478,6 +580,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.junit.jupiter/junit-jupiter", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.junit.jupiter", @@ -492,6 +595,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.assertj/assertj-core@3.23.1", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.assertj", @@ -506,6 +610,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/commons-io/commons-io@2.11.0", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "commons-io", @@ -520,6 +625,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.mockito/mockito-inline@4.8.0", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.mockito", @@ -534,6 +640,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.graalvm.js/js@22.0.0.2", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.graalvm.js", @@ -548,6 +655,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.graalvm.js/js-scriptengine@22.0.0.2", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.graalvm.js", @@ -562,6 +670,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.apache.commons/commons-rng-simple@1.4", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.apache.commons", @@ -576,6 +685,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.openjdk.jmh/jmh-core@1.35", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.openjdk.jmh", @@ -590,6 +700,7 @@ func getCommonsTextExpectedPackages() []pkg.Package { PURL: "pkg:maven/org.openjdk.jmh/jmh-generator-annprocess@1.35", Language: pkg.Java, Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, Metadata: pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: "org.openjdk.jmh", diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/local/contains-child-1/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/contains-child-1/pom.xml new file mode 100644 index 00000000000..18dcd8c4bbf --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/pom/local/contains-child-1/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + contains-child-one + 5 + jar + + + + + my.org + child-one + 1.3.6 + + + + + + + my.org + child-one + + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-2/pom.xml b/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-2/pom.xml index 5864bfa6cf5..5ca8cc4a202 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-2/pom.xml +++ b/syft/pkg/cataloger/java/test-fixtures/pom/local/parent-2/pom.xml @@ -17,6 +17,13 @@ 1 + + + Eclipse Public License v2.0 + https://www.eclipse.org/legal/epl-v20.html + + + diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom index 5864bfa6cf5..3341e805ce9 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/my/org/parent-two/13.7.8/parent-two-13.7.8.pom @@ -8,6 +8,13 @@ 13.7.8 pom + + + Eclipse Public License v2.0 + https://www.eclipse.org/legal/epl-v20.html + + + 3.14.0 4.4 From 9a047e426efbc7d132edd4349de32f3a4fe10c96 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 24 Jul 2024 20:27:02 -0400 Subject: [PATCH 36/42] fix: improve determinism in java archive identification Signed-off-by: Keith Zantow --- go.mod | 2 +- syft/pkg/cataloger/java/archive_parser.go | 8 +++- .../pkg/cataloger/java/archive_parser_test.go | 38 +++++++++++++++++++ .../java/test-fixtures/jar-metadata/Makefile | 7 ++++ .../META-INF/MANIFEST.MF | 2 + .../multiple-matching-1/pom.properties | 3 ++ .../org.multiple/multiple-matching-1/pom.xml | 8 ++++ .../multiple-matching-2/pom.properties | 3 ++ .../org.multiple/multiple-matching-2/pom.xml | 8 ++++ .../multiple-matching-3/pom.properties | 3 ++ .../org.multiple/multiple-matching-3/pom.xml | 8 ++++ 11 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/MANIFEST.MF create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.properties create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.xml create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.properties create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.xml create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.properties create mode 100644 syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.xml diff --git a/go.mod b/go.mod index 8f0a9cef69e..e403ad60c7c 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/BurntSushi/toml v1.4.0 github.com/adrg/xdg v0.5.0 github.com/magiconair/properties v1.8.7 + golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 ) require ( @@ -230,7 +231,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 134e8f18af3..2262d79fb7c 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -6,8 +6,11 @@ import ( "fmt" "os" "path" + "slices" "strings" + "golang.org/x/exp/maps" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -298,7 +301,10 @@ func (j *archiveParser) guessMainPackageNameAndVersionFromPomInfo(ctx context.Co properties, _ := pomPropertiesByParentPath(j.archivePath, j.location, pomPropertyMatches) projects, _ := pomProjectByParentPath(j.archivePath, j.location, pomMatches) - for parentPath, propertiesObj := range properties { + parentPaths := maps.Keys(properties) + slices.Sort(parentPaths) + for _, parentPath := range parentPaths { + propertiesObj := properties[parentPath] if artifactIDMatchesFilename(propertiesObj.ArtifactID, j.fileInfo.name) { pomPropertiesObject = propertiesObj if proj, exists := projects[parentPath]; exists { diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 7d7164196c3..968be857862 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -1386,6 +1386,44 @@ func Test_parseJavaArchive_regressions(t *testing.T) { } } +func Test_deterministicMatchingPomProperties(t *testing.T) { + tests := []struct { + fixture string + expectedName string + expectedVersion string + }{ + { + fixture: "multiple-matching-2.11.5", + expectedName: "multiple-matching-1", + expectedVersion: "2.11.5", + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + fixturePath := generateJavaMetadataJarFixture(t, test.fixture) + + for i := 0; i < 5; i++ { + func() { + fixture, err := os.Open(fixturePath) + require.NoError(t, err) + + parser, cleanupFn, err := newJavaArchiveParser(file.LocationReadCloser{ + Location: file.NewLocation(fixture.Name()), + ReadCloser: fixture, + }, false, ArchiveCatalogerConfig{UseNetwork: false}) + defer cleanupFn() + require.NoError(t, err) + + name, version, _ := parser.guessMainPackageNameAndVersionFromPomInfo(context.TODO()) + require.Equal(t, test.expectedName, name) + require.Equal(t, test.expectedVersion, version) + }() + } + }) + } +} + func assignParent(parent *pkg.Package, childPackages ...pkg.Package) { for i, jp := range childPackages { if v, ok := jp.Metadata.(pkg.JavaArchive); ok { diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile index b6533f1471c..aec64a10821 100644 --- a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile @@ -6,6 +6,7 @@ SBT_JACKSON_CORE = com.fasterxml.jackson.core.jackson-core-2.15.2 OPENSAML_CORE = opensaml-core-3.4.6 API_ALL_SOURCES = api-all-2.0.0-sources SPRING_INSTRUMENTATION = spring-instrumentation-4.3.0-1.0 +MULTIPLE_MATCHING = multiple-matching-2.11.5 $(CACHE_DIR): mkdir -p $(CACHE_DIR) @@ -24,3 +25,9 @@ $(CACHE_DIR)/$(API_ALL_SOURCES).jar: $(CACHE_DIR) $(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar: $(CACHE_DIR) cd $(SPRING_INSTRUMENTATION) && zip -r $(CACHE_PATH)/$(SPRING_INSTRUMENTATION).jar . + +$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar: $(CACHE_DIR) + cd $(SPRING_INSTRUMENTATION) && zip -r $(CACHE_PATH)/$(SPRING_INSTRUMENTATION).jar . + +$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar: $(CACHE_DIR) + cd $(MULTIPLE_MATCHING) && zip -r $(CACHE_PATH)/$(MULTIPLE_MATCHING).jar . diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/MANIFEST.MF b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..525109c7337 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/MANIFEST.MF @@ -0,0 +1,2 @@ +Manifest-Version: 1.0 +Created-By: Multi diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.properties b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.properties new file mode 100644 index 00000000000..43e0d811b73 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.properties @@ -0,0 +1,3 @@ +version=2.11.5 +groupId=org.multiple +artifactId=multiple-matching-1 diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.xml b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.xml new file mode 100644 index 00000000000..1c359dd9aa2 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-1/pom.xml @@ -0,0 +1,8 @@ + + + 4.0.0 + + org.multiple + multiple-matching-1 + 2.11.5 + diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.properties b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.properties new file mode 100644 index 00000000000..6be2acfbd13 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.properties @@ -0,0 +1,3 @@ +version=2.11.5 +groupId=org.multiple +artifactId=multiple-matching-2 \ No newline at end of file diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.xml b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.xml new file mode 100644 index 00000000000..343c49d303d --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-2/pom.xml @@ -0,0 +1,8 @@ + + + 4.0.0 + + org.multiple + multiple-matching-2 + 2.11.5 + diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.properties b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.properties new file mode 100644 index 00000000000..187215b293d --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.properties @@ -0,0 +1,3 @@ +version=2.11.5 +groupId=org.multiple +artifactId=multiple-matching-3 diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.xml b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.xml new file mode 100644 index 00000000000..b428bd69bbd --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/multiple-matching-2.11.5/META-INF/maven/org.multiple/multiple-matching-3/pom.xml @@ -0,0 +1,8 @@ + + + 4.0.0 + + org.multiple + multiple-matching-3 + 2.11.5 + From e8b14f717c28abfd425717eb1fda12bc15d976a0 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Thu, 25 Jul 2024 12:01:17 -0400 Subject: [PATCH 37/42] chore: use structured logging Signed-off-by: Keith Zantow --- syft/pkg/cataloger/java/archive_parser.go | 11 ++- syft/pkg/cataloger/java/maven_resolver.go | 107 ++++++++++++++-------- syft/pkg/cataloger/java/maven_utils.go | 2 +- syft/pkg/cataloger/java/parse_pom_xml.go | 38 +++----- 4 files changed, 91 insertions(+), 67 deletions(-) diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index f02c96e3409..a18ef471af0 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -277,14 +277,14 @@ func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID if parsedPom != nil { pomLicenses, err = j.maven.resolveLicenses(ctx, parsedPom.project) if err != nil { - log.Debugf("error attempting to resolve licenses for %v: %v", j.maven.resolveMavenID(ctx, parsedPom.project), err) + log.WithFields("error", err, "mavenID", j.maven.resolveMavenID(ctx, parsedPom.project)).Debug("error attempting to resolve pom licenses") } } if err == nil && len(pomLicenses) == 0 { pomLicenses, err = j.maven.findLicenses(ctx, groupID, artifactID, version) if err != nil { - log.Debugf("error attempting to resolve licenses for %v: %v", mavenID{groupID, artifactID, version}, err) + log.WithFields("error", err, "mavenID", mavenID{groupID, artifactID, version}).Debug("error attempting to find licenses") } } @@ -292,7 +292,10 @@ func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID // Try removing the last part of the groupId, as sometimes it duplicates the artifactId packages := strings.Split(groupID, ".") groupID = strings.Join(packages[:len(packages)-1], ".") - pomLicenses, _ = j.maven.findLicenses(ctx, groupID, artifactID, version) + pomLicenses, err = j.maven.findLicenses(ctx, groupID, artifactID, version) + if err != nil { + log.WithFields("error", err, "mavenID", mavenID{groupID, artifactID, version}).Debug("error attempting to find sub-group licenses") + } } return toPkgLicenses(&j.location, pomLicenses) @@ -610,7 +613,7 @@ func newPackageFromMavenData(ctx context.Context, r *mavenResolver, pomPropertie } if err != nil { - log.Debugf("error while attempting to resolve licenses for %v: %v", mavenID{pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version}, err) + log.WithFields("error", err, "mavenID", mavenID{pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version}).Debug("error attempting to resolve licenses") } licenses := make([]pkg.License, 0) diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index 6265dfc3032..74ef6f2915e 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -31,28 +31,6 @@ type mavenID struct { Version string } -// resolveMavenID creates a new mavenID from a pom, resolving parent information as necessary -func (r *mavenResolver) resolveMavenID(ctx context.Context, pom *gopom.Project) mavenID { - if pom == nil { - return mavenID{} - } - groupID := r.getPropertyValue(ctx, pom.GroupID, pom) - artifactID := r.getPropertyValue(ctx, pom.ArtifactID, pom) - version := r.getPropertyValue(ctx, pom.Version, pom) - if pom.Parent != nil { - if groupID == "" { - groupID = r.getPropertyValue(ctx, pom.Parent.GroupID, pom) - } - if artifactID == "" { - artifactID = r.getPropertyValue(ctx, pom.Parent.ArtifactID, pom) - } - if version == "" { - version = r.getPropertyValue(ctx, pom.Parent.Version, pom) - } - } - return mavenID{groupID, artifactID, version} -} - func (m mavenID) String() string { return fmt.Sprintf("(groupId: %s artifactId: %s version: %s)", m.GroupID, m.ArtifactID, m.Version) } @@ -88,23 +66,19 @@ func newMavenResolver(fileResolver file.Resolver, cfg ArchiveCatalogerConfig) *m // getPropertyValue gets property values by emulating maven property resolution logic, looking in the project's variables // as well as supporting the project expressions like ${project.parent.groupId}. // Properties which are not resolved result in empty string "" -// -//nolint:gocognit func (r *mavenResolver) getPropertyValue(ctx context.Context, propertyValue *string, resolutionContext ...*gopom.Project) string { if propertyValue == nil { return "" } resolved, err := r.resolveExpression(ctx, resolutionContext, *propertyValue, nil) if err != nil { - log.Debugf("error resolving maven property: %s: %v", *propertyValue, err) + log.WithFields("error", err, "propertyValue", *propertyValue).Debug("error resolving maven property") return "" } return resolved } // resolveExpression resolves an expression, which may be a plain string or a string with ${ property.references } -// -//nolint:gocognit func (r *mavenResolver) resolveExpression(ctx context.Context, resolutionContext []*gopom.Project, expression string, resolving []string) (string, error) { var err error return expressionMatcher.ReplaceAllStringFunc(expression, func(match string) string { @@ -119,8 +93,6 @@ func (r *mavenResolver) resolveExpression(ctx context.Context, resolutionContext } // resolveProperty resolves properties recursively from the root project -// -//nolint:gocognit func (r *mavenResolver) resolveProperty(ctx context.Context, resolutionContext []*gopom.Project, propertyExpression string, resolving []string) (string, error) { // prevent cycles if slices.Contains(resolving, propertyExpression) { @@ -225,6 +197,52 @@ func (r *mavenResolver) resolveProjectProperty(ctx context.Context, resolutionCo return "", nil } +// resolveMavenID creates a new mavenID from a pom, resolving parent information as necessary +func (r *mavenResolver) resolveMavenID(ctx context.Context, pom *gopom.Project) mavenID { + if pom == nil { + return mavenID{} + } + groupID := r.getPropertyValue(ctx, pom.GroupID, pom) + artifactID := r.getPropertyValue(ctx, pom.ArtifactID, pom) + version := r.getPropertyValue(ctx, pom.Version, pom) + if pom.Parent != nil { + if groupID == "" { + groupID = r.getPropertyValue(ctx, pom.Parent.GroupID, pom) + } + if artifactID == "" { + artifactID = r.getPropertyValue(ctx, pom.Parent.ArtifactID, pom) + } + if version == "" { + version = r.getPropertyValue(ctx, pom.Parent.Version, pom) + } + } + return mavenID{groupID, artifactID, version} +} + +// resolveDependencyID creates a new mavenID from a dependency element in a pom, resolving information as necessary +func (r *mavenResolver) resolveDependencyID(ctx context.Context, pom *gopom.Project, dep gopom.Dependency) mavenID { + if pom == nil { + return mavenID{} + } + + groupID := r.getPropertyValue(ctx, dep.GroupID, pom) + artifactID := r.getPropertyValue(ctx, dep.ArtifactID, pom) + version := r.getPropertyValue(ctx, dep.Version, pom) + + var err error + if version == "" { + version, err = r.findInheritedVersion(ctx, pom, groupID, artifactID) + } + + depID := mavenID{groupID, artifactID, version} + + if err != nil { + log.WithFields("error", err, "mavenID", r.resolveMavenID(ctx, pom), "dependencyID", depID) + } + + return depID +} + // findPom gets a pom from cache, local repository, or from a remote Maven repository depending on configuration func (r *mavenResolver) findPom(ctx context.Context, groupID, artifactID, version string) (*gopom.Project, error) { if groupID == "" || artifactID == "" || version == "" { @@ -274,7 +292,8 @@ func (r *mavenResolver) findPomInLocalRepository(groupID, artifactID, version st // check if the directory exists at all, and if not just stop trying to resolve local maven files fi, err := os.Stat(r.cfg.MavenLocalRepositoryDir) if errors.Is(err, os.ErrNotExist) || !fi.IsDir() { - log.Debugf("local maven repository is not a readable directory, stopping local resolution: %v", r.cfg.MavenLocalRepositoryDir) + log.WithFields("error", err, "repositoryDir", r.cfg.MavenLocalRepositoryDir). + Info("local maven repository is not a readable directory, stopping local resolution") r.cfg.UseMavenLocalRepository = false } } @@ -307,7 +326,7 @@ func (r *mavenResolver) findPomInRemoteRepository(ctx context.Context, groupID, if err != nil { return nil, err } - log.Debugf("fetching parent pom from maven repository, at url: %s", requestURL) + log.WithFields("url", requestURL).Info("fetching parent pom from remote maven repository") req, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { @@ -342,6 +361,10 @@ func (r *mavenResolver) findPomInRemoteRepository(ctx context.Context, groupID, return pom, nil } +// cacheResolveReader attempts to get a reader from cache, otherwise caches the contents of the resolve() function. +// this function is guaranteed to return an unread reader for the correct contents. +// NOTE: this could be promoted to the internal cache package as a specialized version of the cache.Resolver +// if there are more users of this functionality func (r *mavenResolver) cacheResolveReader(key string, resolve func() (io.ReadCloser, error)) (io.Reader, error) { reader, err := r.cache.Read(key) if err == nil && reader != nil { @@ -426,12 +449,14 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, pom *gopom.Pro depPom, err := r.findPom(ctx, depGroupID, depArtifactID, depVersion) if err != nil || depPom == nil { - log.Debugf("unable to find imported pom looking for managed dependencies for: %v for dependency: %v: %v", r.resolveMavenID(ctx, pom), mavenID{depGroupID, depArtifactID, depVersion}, err) + log.WithFields("error", err, "mavenID", r.resolveMavenID(ctx, pom), "dependencyID", mavenID{depGroupID, depArtifactID, depVersion}). + Debug("unable to find imported pom looking for managed dependencies") continue } version, err = r.findInheritedVersion(ctx, depPom, groupID, artifactID, resolutionContext...) if err != nil { - log.Debugf("error calling findInheritedVersion for: %v for dependency: %v: %v", r.resolveMavenID(ctx, pom), mavenID{depGroupID, depArtifactID, depVersion}, err) + log.WithFields("error", err, "mavenID", r.resolveMavenID(ctx, pom), "dependencyID", mavenID{depGroupID, depArtifactID, depVersion}). + Debug("error during findInheritedVersion") } if version != "" { return version, nil @@ -537,26 +562,30 @@ func (r *mavenResolver) findParentPomByRelativePath(ctx context.Context, pom *go p = path.Clean(p) parentLocations, err := r.fileResolver.FilesByPath(p) if err != nil || len(parentLocations) == 0 { - log.Debugf("parent not found in by relative path for: %v looking for: %v at %v err: %v", r.resolveMavenID(ctx, pom), parentID, relativePath, err) + log.WithFields("error", err, "mavenID", r.resolveMavenID(ctx, pom), "parentID", parentID, "relativePath", relativePath). + Trace("parent pom not found by relative path") return nil } parentLocation := parentLocations[0] parentContents, err := r.fileResolver.FileContentsByLocation(parentLocation) if err != nil || parentContents == nil { - log.Debugf("unable to get parent by relative path for: %v parent: %v at %v err: %v", r.resolveMavenID(ctx, pom), parentID, parentLocation, err) + log.WithFields("error", err, "mavenID", r.resolveMavenID(ctx, pom), "parentID", parentID, "parentLocation", parentLocation). + Debug("unable to get contents of parent pom by relative path") return nil } defer internal.CloseAndLogError(parentContents, parentLocation.RealPath) parentPom, err := decodePomXML(parentContents) if err != nil || parentPom == nil { - log.Debugf("unable to parse parent by relative path for: %v parent: %v at %v err: %v", r.resolveMavenID(ctx, pom), parentID, parentLocation, err) + log.WithFields("error", err, "mavenID", r.resolveMavenID(ctx, pom), "parentID", parentID, "parentLocation", parentLocation). + Debug("unable to parse parent pom") return nil } - // ensure ids match + // ensure parent matches newParentID := r.resolveMavenID(ctx, parentPom) - if newParentID != parentID { - log.Debugf("parent IDs do not match resolving parent by relative path for: %v parent: %v at %v, got: %v", r.resolveMavenID(ctx, pom), parentID, parentLocation, newParentID) + if newParentID.ArtifactID != parentID.ArtifactID { + log.WithFields("newParentID", newParentID, "mavenID", r.resolveMavenID(ctx, pom), "parentID", parentID, "parentLocation", parentLocation). + Debug("parent IDs do not match resolving parent by relative path") return nil } diff --git a/syft/pkg/cataloger/java/maven_utils.go b/syft/pkg/cataloger/java/maven_utils.go index 101b74212a4..9d365e151d6 100644 --- a/syft/pkg/cataloger/java/maven_utils.go +++ b/syft/pkg/cataloger/java/maven_utils.go @@ -44,7 +44,7 @@ func getSettingsXMLLocalRepository(settingsXML io.Reader) string { s := settings{} err := xml.NewDecoder(settingsXML).Decode(&s) if err != nil { - log.Debugf("unable to read maven settings.xml: %v", err) + log.WithFields("error", err).Debug("unable to read maven settings.xml") } return s.LocalRepository } diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index b9df0ee7001..d9fe4b2c25a 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -45,7 +45,7 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver for _, pomLocation := range locations { pom, err := readPomFromLocation(fileResolver, pomLocation) if err != nil || pom == nil { - log.Debugf("error while getting contents for: %v %v", pomLocation.RealPath, err) + log.WithFields("error", err, "pomLocation", pomLocation).Debug("error while reading pom") continue } @@ -75,13 +75,11 @@ func readPomFromLocation(fileResolver file.Resolver, pomLocation file.Location) func processPomXML(ctx context.Context, r *mavenResolver, pom *gopom.Project, loc file.Location) []pkg.Package { var pkgs []pkg.Package + pomID := r.resolveMavenID(ctx, pom) for _, dep := range pomDependencies(pom) { - id := mavenID{ - r.getPropertyValue(ctx, dep.GroupID, pom), - r.getPropertyValue(ctx, dep.ArtifactID, pom), - r.getPropertyValue(ctx, dep.Version, pom), - } - log.Tracef("adding dependency to SBOM: %v from: %v", id, r.resolveMavenID(ctx, pom)) + depID := r.resolveDependencyID(ctx, pom, dep) + log.WithFields("pomLocation", loc, "mavenID", pomID, "dependencyID", depID).Trace("adding maven pom dependency") + p, err := newPackageFromDependency( ctx, r, @@ -90,7 +88,7 @@ func processPomXML(ctx context.Context, r *mavenResolver, pom *gopom.Project, lo loc.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) if err != nil { - log.Debugf("error adding dependency %v: %v", id, err) + log.WithFields("error", err, "pomLocation", loc, "mavenID", pomID, "dependencyID", depID).Debugf("error adding dependency") } if p == nil { continue @@ -120,29 +118,23 @@ func newPomProject(ctx context.Context, r *mavenResolver, path string, pom *gopo } func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom.Project, dep gopom.Dependency, locations ...file.Location) (*pkg.Package, error) { - groupID := r.getPropertyValue(ctx, dep.GroupID, pom) - artifactID := r.getPropertyValue(ctx, dep.ArtifactID, pom) - version := r.getPropertyValue(ctx, dep.Version, pom) - - var err error - if version == "" { - version, err = r.findInheritedVersion(ctx, pom, groupID, artifactID) - } + id := r.resolveDependencyID(ctx, pom, dep) m := pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ - GroupID: groupID, - ArtifactID: artifactID, + GroupID: id.GroupID, + ArtifactID: id.ArtifactID, Scope: r.getPropertyValue(ctx, dep.Scope, pom), }, } + var err error var licenses []pkg.License - dependencyPom, depErr := r.findPom(ctx, groupID, artifactID, version) + dependencyPom, depErr := r.findPom(ctx, id.GroupID, id.ArtifactID, id.Version) if depErr != nil { - log.Debugf("error getting licenses for %s: %v", mavenID{groupID, artifactID, version}, err) err = errors.Join(err, depErr) } + if dependencyPom != nil { depLicenses, _ := r.resolveLicenses(ctx, dependencyPom) for _, license := range depLicenses { @@ -151,11 +143,11 @@ func newPackageFromDependency(ctx context.Context, r *mavenResolver, pom *gopom. } p := &pkg.Package{ - Name: artifactID, - Version: version, + Name: id.ArtifactID, + Version: id.Version, Locations: file.NewLocationSet(locations...), Licenses: pkg.NewLicenseSet(licenses...), - PURL: packageURL(artifactID, version, m), + PURL: packageURL(id.ArtifactID, id.Version, m), Language: pkg.Java, Type: pkg.JavaPkg, // TODO: should we differentiate between packages from jar/war/zip versus packages from a pom.xml that were not installed yet? FoundBy: pomCatalogerName, From 80253f77fa8b9fa119ee257abef5b29e1b678c57 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Thu, 25 Jul 2024 12:04:16 -0400 Subject: [PATCH 38/42] chore: use structured logging Signed-off-by: Keith Zantow --- syft/file/license.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syft/file/license.go b/syft/file/license.go index fe9b5b93f29..08d77e052e8 100644 --- a/syft/file/license.go +++ b/syft/file/license.go @@ -21,7 +21,7 @@ type LicenseEvidence struct { func NewLicense(value string) License { spdxExpression, err := license.ParseExpression(value) if err != nil { - log.Tracef("unable to parse license expression: '%s', error: '%v'", value, err) + log.WithFields("error", err, "value", value).Trace("unable to parse license expression") } return License{ From 697b4e16ec2ca73dccf85827fa22eb476a130bd7 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Mon, 29 Jul 2024 17:11:03 -0400 Subject: [PATCH 39/42] chore: don't trim existing pom Signed-off-by: Keith Zantow --- .../3.4.6/opensaml-parent-3.4.6.pom | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom index b23c101a0c7..e3e2be465ec 100644 --- a/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom +++ b/syft/pkg/cataloger/java/test-fixtures/pom/maven-repo/org/opensaml/opensaml-parent/3.4.6/opensaml-parent-3.4.6.pom @@ -10,12 +10,43 @@ OpenSAML + + A library for creating, reading, writing and performing some processing of SAML messages. + + For further information see https://wiki.shibboleth.net/confluence/display/OS30/Home + org.opensaml opensaml-parent 3.4.6 pom + + ../opensaml-core + ../opensaml-storage-api + ../opensaml-security-api + ../opensaml-xmlsec-api + ../opensaml-messaging-api + ../opensaml-soap-api + ../opensaml-saml-api + ../opensaml-xacml-api + ../opensaml-xacml-saml-api + + ../opensaml-storage-impl + ../opensaml-security-impl + ../opensaml-xmlsec-impl + ../opensaml-messaging-impl + ../opensaml-soap-impl + ../opensaml-saml-impl + ../opensaml-xacml-impl + ../opensaml-xacml-saml-impl + ../opensaml-profile-api + ../opensaml-profile-impl + + ../opensaml-bom + ../opensaml-tests-bom + + 7.5.2 5.4.2 @@ -24,7 +55,11 @@ ${opensaml-parent.site.url}${project.artifactId} + + + + net.shibboleth.utilities java-support @@ -35,6 +70,11 @@ commons-codec + + + + + net.shibboleth.utilities java-support @@ -50,8 +90,12 @@ + + javax.xml.bind jaxb-api @@ -79,6 +123,11 @@ 2.12.3 + + + + + net.shibboleth.ext spring-extensions @@ -102,4 +151,36 @@ + + + + org.apache.maven.plugins + maven-site-plugin + + + attach-descriptor + + attach-descriptor + + + + + ../opensaml-parent/src/site + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${automatic.module.name} + + + + + + + \ No newline at end of file From 084e1f7ef951194296432ff2f0cb78c4eb463940 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Mon, 29 Jul 2024 18:10:27 -0400 Subject: [PATCH 40/42] chore: reorganize test utils Signed-off-by: Keith Zantow --- .../pkg/cataloger/java/archive_parser_test.go | 64 ++++++------------- .../pkg/cataloger/java/maven_resolver_test.go | 38 +++++++++-- syft/pkg/cataloger/java/parse_pom_xml_test.go | 4 +- 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 0793f2ea31d..c76df2c5ad3 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "net/http" "os" "os/exec" "path/filepath" @@ -21,7 +20,6 @@ import ( "github.com/stretchr/testify/require" "github.com/vifraa/gopom" - "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/license" @@ -29,48 +27,8 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) -func generateJavaBuildFixture(t *testing.T, fixturePath string) { - if _, err := os.Stat(fixturePath); !os.IsNotExist(err) { - // fixture already exists... - return - } - - makeTask := strings.TrimPrefix(fixturePath, "test-fixtures/java-builds/") - t.Logf(color.Bold.Sprintf("Generating Fixture from 'make %s'", makeTask)) - - cwd, err := os.Getwd() - if err != nil { - t.Errorf("unable to get cwd: %+v", err) - } - - cmd := exec.Command("make", makeTask) - cmd.Dir = filepath.Join(cwd, "test-fixtures/java-builds/") - - run(t, cmd) -} - -func generateMockMavenHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - // Set the Content-Type header to indicate that the response is XML - w.Header().Set("Content-Type", "application/xml") - // Copy the file's content to the response writer - f, err := os.Open(responseFixture) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer internal.CloseAndLogError(f, responseFixture) - _, err = io.Copy(w, f) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - func TestSearchMavenForLicenses(t *testing.T) { - url := testRepo(t, "test-fixtures/pom/maven-repo") + url := mockMavenRepo(t) tests := []struct { name string @@ -1395,6 +1353,26 @@ func assignParent(parent *pkg.Package, childPackages ...pkg.Package) { } } +func generateJavaBuildFixture(t *testing.T, fixturePath string) { + if _, err := os.Stat(fixturePath); !os.IsNotExist(err) { + // fixture already exists... + return + } + + makeTask := strings.TrimPrefix(fixturePath, "test-fixtures/java-builds/") + t.Logf(color.Bold.Sprintf("Generating Fixture from 'make %s'", makeTask)) + + cwd, err := os.Getwd() + if err != nil { + t.Errorf("unable to get cwd: %+v", err) + } + + cmd := exec.Command("make", makeTask) + cmd.Dir = filepath.Join(cwd, "test-fixtures/java-builds/") + + run(t, cmd) +} + func generateJavaMetadataJarFixture(t *testing.T, fixtureName string) string { fixturePath := filepath.Join("test-fixtures/jar-metadata/cache/", fixtureName+".jar") if _, err := os.Stat(fixturePath); !os.IsNotExist(err) { diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index b3fe70a78b8..afe70792fac 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -2,6 +2,7 @@ package java import ( "context" + "io" "net/http" "net/http/httptest" "os" @@ -208,7 +209,7 @@ func Test_mavenResolverLocal(t *testing.T) { } func Test_mavenResolverRemote(t *testing.T) { - url := testRepo(t, "test-fixtures/pom/maven-repo") + url := mockMavenRepo(t) tests := []struct { groupID string @@ -280,8 +281,17 @@ func Test_relativePathParent(t *testing.T) { require.Equal(t, "3", got) } -// testRepo starts a remote maven repo serving all the pom files found in the given directory -func testRepo(t *testing.T, dir string) (url string) { +// mockMavenRepo starts a remote maven repo serving all the pom files found in test-fixtures/pom/maven-repo +func mockMavenRepo(t *testing.T) (url string) { + t.Helper() + + return mockMavenRepoAt(t, "test-fixtures/pom/maven-repo") +} + +// mockMavenRepoAt starts a remote maven repo serving all the pom files found in the given directory +func mockMavenRepoAt(t *testing.T, dir string) (url string) { + t.Helper() + // mux is the HTTP request multiplexer used with the test server. mux := http.NewServeMux() @@ -302,8 +312,28 @@ func testRepo(t *testing.T, dir string) (url string) { fullPath, err := filepath.Abs(filepath.Join(dir, match)) require.NoError(t, err) match = "/" + filepath.ToSlash(match) - mux.HandleFunc(match, generateMockMavenHandler(fullPath)) + mux.HandleFunc(match, mockMavenHandler(fullPath)) } return server.URL } + +func mockMavenHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + // Set the Content-Type header to indicate that the response is XML + w.Header().Set("Content-Type", "application/xml") + // Copy the file's content to the response writer + f, err := os.Open(responseFixture) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer internal.CloseAndLogError(f, responseFixture) + _, err = io.Copy(w, f) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 3068e2a1f6e..8344e252007 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -214,7 +214,7 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { } func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { - url := testRepo(t, "test-fixtures/pom/maven-repo") + url := mockMavenRepo(t) // Using the local repository, the version of junit-jupiter will be resolved expectedPackages := getCommonsTextExpectedPackages() @@ -433,7 +433,7 @@ func Test_cleanDescription(t *testing.T) { } func Test_resolveLicenses(t *testing.T) { - mavenURL := testRepo(t, "test-fixtures/pom/maven-repo") + mavenURL := mockMavenRepo(t) localM2 := "test-fixtures/pom/maven-repo" localDir := "test-fixtures/pom/local" containingDir := "test-fixtures/pom/local/contains-child-1" From 7837e2647428e7a4401f5dfa7c404f743c198231 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 30 Jul 2024 23:32:38 -0400 Subject: [PATCH 41/42] fix: properly respect max parent depth, default to unlimited Signed-off-by: Keith Zantow --- cmd/syft/internal/options/java.go | 2 +- .../pkg/cataloger/java/archive_parser_test.go | 1 - syft/pkg/cataloger/java/config.go | 6 ++--- syft/pkg/cataloger/java/maven_resolver.go | 9 ++++--- .../pkg/cataloger/java/maven_resolver_test.go | 26 ++++++++++++++++--- syft/pkg/cataloger/java/parse_pom_xml_test.go | 6 ----- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/cmd/syft/internal/options/java.go b/cmd/syft/internal/options/java.go index 89b0399df3b..8894c760b75 100644 --- a/cmd/syft/internal/options/java.go +++ b/cmd/syft/internal/options/java.go @@ -34,7 +34,7 @@ func (o *javaConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { a parent or imported pom file is not found in the local maven repository. the pom files are downloaded from the remote Maven repository at 'maven-url'`) descriptions.Add(&o.MavenURL, `maven repository to use, defaults to Maven central`) - descriptions.Add(&o.MaxParentRecursiveDepth, `depth to recursively resolve parent POMs`) + descriptions.Add(&o.MaxParentRecursiveDepth, `depth to recursively resolve parent POMs, no limit if <= 0`) descriptions.Add(&o.UseMavenLocalRepository, `use the local Maven repository to retrieve pom files. When Maven is installed and was previously used for building the software that is being scanned, then most pom files will be available in this repository on the local file system. this greatly speeds up scans. when all pom files are available diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index c76df2c5ad3..d5055d44fe3 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -45,7 +45,6 @@ func TestSearchMavenForLicenses(t *testing.T) { UseNetwork: true, UseMavenLocalRepository: false, MavenBaseURL: url, - MaxParentRecursiveDepth: 5, }, expectedLicenses: []pkg.License{ { diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index df48f12bfde..29096d59bff 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -20,7 +20,7 @@ func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { UseMavenLocalRepository: false, MavenLocalRepositoryDir: defaultMavenLocalRepoDir(), MavenBaseURL: mavenBaseURL, - MaxParentRecursiveDepth: 10, + MaxParentRecursiveDepth: 0, // unlimited } } @@ -47,9 +47,7 @@ func (j ArchiveCatalogerConfig) WithMavenBaseURL(input string) ArchiveCatalogerC } func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSearchConfig, maxDepth int) ArchiveCatalogerConfig { - if maxDepth > 0 { - j.MaxParentRecursiveDepth = maxDepth - } + j.MaxParentRecursiveDepth = maxDepth j.ArchiveSearchConfig = search return j } diff --git a/syft/pkg/cataloger/java/maven_resolver.go b/syft/pkg/cataloger/java/maven_resolver.go index 74ef6f2915e..5fddccc3191 100644 --- a/syft/pkg/cataloger/java/maven_resolver.go +++ b/syft/pkg/cataloger/java/maven_resolver.go @@ -114,7 +114,10 @@ func (r *mavenResolver) resolveProperty(ctx context.Context, resolutionContext [ for _, pom := range resolutionContext { current := pom - for current != nil { + for parentDepth := 0; current != nil; parentDepth++ { + if r.cfg.MaxParentRecursiveDepth > 0 && parentDepth > r.cfg.MaxParentRecursiveDepth { + return "", fmt.Errorf("maximum parent recursive depth (%v) reached resolving property: %v", r.cfg.MaxParentRecursiveDepth, propertyExpression) + } if current.Properties != nil && current.Properties.Entries != nil { if value, ok := current.Properties.Entries[propertyExpression]; ok { return r.resolveExpression(ctx, resolutionContext, value, resolving) // property values can contain expressions @@ -421,7 +424,7 @@ func (r *mavenResolver) findInheritedVersion(ctx context.Context, pom *gopom.Pro if pom == nil { return "", fmt.Errorf("nil pom provided to findInheritedVersion") } - if len(resolutionContext) >= r.cfg.MaxParentRecursiveDepth { + if r.cfg.MaxParentRecursiveDepth > 0 && len(resolutionContext) > r.cfg.MaxParentRecursiveDepth { return "", fmt.Errorf("maximum depth reached attempting to resolve version for: %s:%s at: %v", groupID, artifactID, r.resolveMavenID(ctx, pom)) } if slices.Contains(resolutionContext, pom) { @@ -509,7 +512,7 @@ func (r *mavenResolver) resolveLicenses(ctx context.Context, pom *gopom.Project, if slices.Contains(processing, id) { return nil, fmt.Errorf("cycle detected resolving licenses for: %v", id) } - if len(processing) > r.cfg.MaxParentRecursiveDepth { + if r.cfg.MaxParentRecursiveDepth > 0 && len(processing) > r.cfg.MaxParentRecursiveDepth { return nil, fmt.Errorf("maximum parent recursive depth (%v) reached: %v", r.cfg.MaxParentRecursiveDepth, processing) } diff --git a/syft/pkg/cataloger/java/maven_resolver_test.go b/syft/pkg/cataloger/java/maven_resolver_test.go index afe70792fac..d778efb6991 100644 --- a/syft/pkg/cataloger/java/maven_resolver_test.go +++ b/syft/pkg/cataloger/java/maven_resolver_test.go @@ -171,30 +171,51 @@ func Test_mavenResolverLocal(t *testing.T) { require.NoError(t, err) tests := []struct { + name string groupID string artifactID string version string + maxDepth int expression string expected string wantErr require.ErrorAssertionFunc }{ { + name: "artifact id with variable from 2nd parent", groupID: "my.org", artifactID: "child-one", version: "1.3.6", expression: "${project.one}", expected: "1", }, + { + name: "depth limited large enough", + groupID: "my.org", + artifactID: "child-one", + version: "1.3.6", + expression: "${project.one}", + expected: "1", + maxDepth: 2, + }, + { + name: "depth limited should not resolve", + groupID: "my.org", + artifactID: "child-one", + version: "1.3.6", + expression: "${project.one}", + expected: "", + maxDepth: 1, + }, } for _, test := range tests { - t.Run(test.artifactID, func(t *testing.T) { + t.Run(test.name, func(t *testing.T) { ctx := context.Background() r := newMavenResolver(nil, ArchiveCatalogerConfig{ UseNetwork: false, UseMavenLocalRepository: true, MavenLocalRepositoryDir: dir, - MaxParentRecursiveDepth: 5, + MaxParentRecursiveDepth: test.maxDepth, }) pom, err := r.findPom(ctx, test.groupID, test.artifactID, test.version) if test.wantErr != nil { @@ -235,7 +256,6 @@ func Test_mavenResolverRemote(t *testing.T) { UseNetwork: true, UseMavenLocalRepository: false, MavenBaseURL: url, - MaxParentRecursiveDepth: 5, }) pom, err := r.findPom(ctx, test.groupID, test.artifactID, test.version) if test.wantErr != nil { diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 8344e252007..45650049072 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -206,7 +206,6 @@ func Test_parseCommonsTextPomXMLProjectWithLocalRepository(t *testing.T) { }, UseMavenLocalRepository: true, MavenLocalRepositoryDir: "test-fixtures/pom/maven-repo", - MaxParentRecursiveDepth: 5, }) pkgtest.TestCataloger(t, test.dir, cat, test.expected, nil) }) @@ -258,7 +257,6 @@ func Test_parseCommonsTextPomXMLProjectWithNetwork(t *testing.T) { UseNetwork: true, MavenBaseURL: url, UseMavenLocalRepository: false, - MaxParentRecursiveDepth: 5, }) pkgtest.TestCataloger(t, test.dir, cat, test.expected, nil) }) @@ -461,7 +459,6 @@ func Test_resolveLicenses(t *testing.T) { UseNetwork: false, MavenLocalRepositoryDir: "", MavenBaseURL: "", - MaxParentRecursiveDepth: 10, }, expected: nil, }, @@ -471,7 +468,6 @@ func Test_resolveLicenses(t *testing.T) { cfg: ArchiveCatalogerConfig{ UseMavenLocalRepository: false, UseNetwork: false, - MaxParentRecursiveDepth: 10, }, expected: expectedLicenses, }, @@ -483,7 +479,6 @@ func Test_resolveLicenses(t *testing.T) { MavenLocalRepositoryDir: localM2, UseNetwork: false, MavenBaseURL: "", - MaxParentRecursiveDepth: 10, }, expected: expectedLicenses, }, @@ -494,7 +489,6 @@ func Test_resolveLicenses(t *testing.T) { UseMavenLocalRepository: false, UseNetwork: true, MavenBaseURL: mavenURL, - MaxParentRecursiveDepth: 10, }, expected: expectedLicenses, }, From a2a695fdc7c65977d0c993ba17144fd85c12388e Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Sun, 4 Aug 2024 00:17:59 -0400 Subject: [PATCH 42/42] chore: pr feedback Signed-off-by: Keith Zantow --- syft/pkg/license.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syft/pkg/license.go b/syft/pkg/license.go index bb070299e6f..ae311a13c35 100644 --- a/syft/pkg/license.go +++ b/syft/pkg/license.go @@ -70,7 +70,7 @@ func NewLicenseFromType(value string, t license.Type) License { var err error spdxExpression, err = license.ParseExpression(value) if err != nil { - log.Tracef("unable to parse license expression: '%s', error: '%v'", value, err) + log.WithFields("error", err, "expression", value).Trace("unable to parse license expression") } }