From 4305a71df6f8e7f32cbee0b1c9a99094d30d6766 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 17 Jun 2024 14:21:19 +0600 Subject: [PATCH 1/4] test: add test --- pkg/dependency/parser/java/pom/parse.go | 6 ++ pkg/dependency/parser/java/pom/parse_test.go | 61 ++++++++++++++++++- ... => example-dependency-1.2.3-SNAPSHOT.pom} | 0 ...e-dependency-2.17.0-20240312.035235-10.pom | 23 +++++++ .../2.17.0-SNAPSHOT/maven-metadata.xml | 35 +++++++++++ .../snapshot/with-maven-metadata/pom.xml | 20 ++++++ 6 files changed, 144 insertions(+), 1 deletion(-) rename pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/{example-dependency-1.2.3.pom => example-dependency-1.2.3-SNAPSHOT.pom} (100%) create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml create mode 100644 pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index bf8df2ad1c0a..02d93d0ff408 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -48,6 +48,12 @@ func WithReleaseRemoteRepos(repos []string) option { } } +func WithSnapshotRemoteRepos(repos []string) option { + return func(opts *options) { + opts.snapshotRemoteRepos = repos + } +} + type Parser struct { logger *log.Logger rootPath string diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go index 1207f32adcf7..3ccb550c66a8 100644 --- a/pkg/dependency/parser/java/pom/parse_test.go +++ b/pkg/dependency/parser/java/pom/parse_test.go @@ -143,6 +143,13 @@ func TestPom_Parse(t *testing.T) { }, }, }, + { + ID: "org.example:example-api:2.0.0", + Name: "org.example:example-api", + Version: "2.0.0", + Licenses: []string{"The Apache Software License, Version 2.0"}, + Relationship: ftypes.RelationshipIndirect, + }, }, wantDeps: []ftypes.Dependency{ { @@ -151,6 +158,58 @@ func TestPom_Parse(t *testing.T) { "org.example:example-dependency:1.2.3-SNAPSHOT", }, }, + { + ID: "org.example:example-dependency:1.2.3-SNAPSHOT", + DependsOn: []string{ + "org.example:example-api:2.0.0", + }, + }, + }, + }, + { + name: "snapshot repository with maven-metadata.xml", + inputFile: filepath.Join("testdata", "snapshot", "with-maven-metadata", "pom.xml"), + local: false, + want: []ftypes.Package{ + { + ID: "com.example:happy:1.0.0", + Name: "com.example:happy", + Version: "1.0.0", + Relationship: ftypes.RelationshipRoot, + }, + { + ID: "org.example:example-dependency:2.17.0-SNAPSHOT", + Name: "org.example:example-dependency", + Version: "2.17.0-SNAPSHOT", + Relationship: ftypes.RelationshipDirect, + Locations: ftypes.Locations{ + { + StartLine: 14, + EndLine: 18, + }, + }, + }, + { + ID: "org.example:example-api:2.0.0", + Name: "org.example:example-api", + Version: "2.0.0", + Licenses: []string{"The Apache Software License, Version 2.0"}, + Relationship: ftypes.RelationshipIndirect, + }, + }, + wantDeps: []ftypes.Dependency{ + { + ID: "com.example:happy:1.0.0", + DependsOn: []string{ + "org.example:example-dependency:1.2.3-SNAPSHOT", + }, + }, + { + ID: "org.example:example-dependency:1.2.3-SNAPSHOT", + DependsOn: []string{ + "org.example:example-api:2.0.0", + }, + }, }, }, { @@ -1404,7 +1463,7 @@ func TestPom_Parse(t *testing.T) { remoteRepos = []string{ts.URL} } - p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) + p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithSnapshotRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) gotPkgs, gotDeps, err := p.Parse(f) if tt.wantErr != "" { diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3-SNAPSHOT.pom similarity index 100% rename from pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom rename to pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3-SNAPSHOT.pom diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom new file mode 100644 index 000000000000..0ecec117f873 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom @@ -0,0 +1,23 @@ + + + + 4.0.0 + + org.example + example-dependency + 2.17.0-SNAPSHOT + + jar + Example API Dependency + The example API + + + + org.example + example-api + 2.0.0 + + + + \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml new file mode 100644 index 000000000000..258de9db99c5 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml @@ -0,0 +1,35 @@ + + org.example + example-dependency + + 20240312035235 + + 20240312.035235 + 10 + + + + sources + jar + 2.17.0-20240312.035235-10 + 20240312035235 + + + module + 2.17.0-20240312.035235-10 + 20240312035235 + + + jar + 2.17.0-20240312.035235-10 + 20240312035235 + + + pom + 2.17.0-20240312.035235-10 + 20240312035235 + + + + 2.17.0-SNAPSHOT + \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml b/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml new file mode 100644 index 000000000000..7e4c2144d417 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + + com.example + happy + 1.0.0 + + happy + Example + + + + + org.example + example-dependency + 2.17.0-SNAPSHOT + + + From f6183cbe771bdb2170c4470f0cadccb72e9a850c Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 17 Jun 2024 15:02:33 +0600 Subject: [PATCH 2/4] feat: add maven-metadata struct --- pkg/dependency/parser/java/pom/metadata.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 pkg/dependency/parser/java/pom/metadata.go diff --git a/pkg/dependency/parser/java/pom/metadata.go b/pkg/dependency/parser/java/pom/metadata.go new file mode 100644 index 000000000000..5bf6870c45df --- /dev/null +++ b/pkg/dependency/parser/java/pom/metadata.go @@ -0,0 +1,15 @@ +package pom + +type Metadata struct { + GroupId string `xml:"groupId"` + ArtifactId string `xml:"artifactId"` + Versioning struct { + SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"` + } `xml:"versioning"` + Version string `xml:"version"` +} + +type SnapshotVersion struct { + Extension string `xml:"extension"` + Value string `xml:"value"` +} From 4d67e6f5f8fa2a72090e989c226816b2db9019c8 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 17 Jun 2024 15:04:32 +0600 Subject: [PATCH 3/4] feat(pom): check maven-metadata.xml to find pom file name --- pkg/dependency/parser/java/pom/parse.go | 74 ++++++++++++++++++-- pkg/dependency/parser/java/pom/parse_test.go | 4 +- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index 02d93d0ff408..c8fda40967bf 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -14,6 +14,7 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/samber/lo" + "golang.org/x/exp/slices" "golang.org/x/net/html/charset" "golang.org/x/xerrors" @@ -654,7 +655,18 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( // try all remoteRepositories for _, repo := range remoteRepos { - fetched, err := p.fetchPOMFromRemoteRepository(repo, paths) + repoPaths := slices.Clone(paths) // Clone slice to avoid overwriting last element of `paths` + if snapshot { + pomFileName, err := p.fetchPomFileNameFromMavenMetadata(repo, repoPaths) + if err != nil { + return nil, xerrors.Errorf("fetch maven-metadata.xml error: %w", err) + } + // Use file name from `maven-metadata.xml` if it exists + if pomFileName != "" { + repoPaths[len(repoPaths)-1] = pomFileName + } + } + fetched, err := p.fetchPOMFromRemoteRepository(repo, repoPaths) if err != nil { return nil, xerrors.Errorf("fetch repository error: %w", err) } else if fetched == nil { @@ -665,7 +677,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories") } -func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) { +func (p *Parser) repoRequest(repo string, paths []string) (*http.Request, error) { repoURL, err := url.Parse(repo) if err != nil { p.logger.Error("URL parse error", log.String("repo", repo)) @@ -676,7 +688,6 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom repoURL.Path = path.Join(paths...) logger := p.logger.With(log.String("host", repoURL.Host), log.String("path", repoURL.Path)) - client := &http.Client{} req, err := http.NewRequest("GET", repoURL.String(), http.NoBody) if err != nil { logger.Debug("HTTP request failed") @@ -687,9 +698,54 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom req.SetBasicAuth(repoURL.User.Username(), password) } + return req, nil +} + +// fetchPomFileNameFromMavenMetadata fetches `maven-metadata.xml` file to detect file name of pom file. +func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) (string, error) { + // Overwrite pom file name to `maven-metadata.xml` + mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths` + mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml") + + req, err := p.repoRequest(repo, mavenMetadataPaths) + if err != nil { + return "", xerrors.Errorf("unable to create request for maven-metadata.xml file") + } + + client := &http.Client{} resp, err := client.Do(req) if err != nil || resp.StatusCode != http.StatusOK { - logger.Debug("Failed to fetch") + p.logger.Debug("Failed to fetch", log.String("url", req.URL.String())) + return "", nil + } + defer resp.Body.Close() + + mavenMetadata, err := parseMavenMetadata(resp.Body) + if err != nil { + return "", xerrors.Errorf("failed to parse the remote POM: %w", err) + } + + var pomFileName string + for _, sv := range mavenMetadata.Versioning.SnapshotVersions { + if sv.Extension == "pom" { + // mavenMetadataPaths[len(mavenMetadataPaths)-3] is artifactID + pomFileName = fmt.Sprintf("%s-%s.pom", mavenMetadataPaths[len(mavenMetadataPaths)-3], sv.Value) + } + } + + return pomFileName, nil +} + +func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) { + req, err := p.repoRequest(repo, paths) + if err != nil { + return nil, xerrors.Errorf("unable to create request for pom file") + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + p.logger.Debug("Failed to fetch", log.String("url", req.URL.String())) return nil, nil } defer resp.Body.Close() @@ -715,6 +771,16 @@ func parsePom(r io.Reader) (*pomXML, error) { return parsed, nil } +func parseMavenMetadata(r io.Reader) (*Metadata, error) { + parsed := &Metadata{} + decoder := xml.NewDecoder(r) + decoder.CharsetReader = charset.NewReaderLabel + if err := decoder.Decode(parsed); err != nil { + return nil, xerrors.Errorf("xml decode error: %w", err) + } + return parsed, nil +} + func packageID(name, version string) string { return dependency.ID(ftypes.Pom, name, version) } diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go index 3ccb550c66a8..15740d599eb9 100644 --- a/pkg/dependency/parser/java/pom/parse_test.go +++ b/pkg/dependency/parser/java/pom/parse_test.go @@ -201,11 +201,11 @@ func TestPom_Parse(t *testing.T) { { ID: "com.example:happy:1.0.0", DependsOn: []string{ - "org.example:example-dependency:1.2.3-SNAPSHOT", + "org.example:example-dependency:2.17.0-SNAPSHOT", }, }, { - ID: "org.example:example-dependency:1.2.3-SNAPSHOT", + ID: "org.example:example-dependency:2.17.0-SNAPSHOT", DependsOn: []string{ "org.example:example-api:2.0.0", }, From 8c54d0ac8bf2c26e98d19e86a0d16c92f8978f33 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 18 Jun 2024 09:51:26 +0600 Subject: [PATCH 4/4] refactor --- pkg/dependency/parser/java/pom/metadata.go | 14 ++++++++------ pkg/dependency/parser/java/pom/parse.go | 10 +++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/dependency/parser/java/pom/metadata.go b/pkg/dependency/parser/java/pom/metadata.go index 5bf6870c45df..0a35e9e4f556 100644 --- a/pkg/dependency/parser/java/pom/metadata.go +++ b/pkg/dependency/parser/java/pom/metadata.go @@ -1,12 +1,14 @@ package pom type Metadata struct { - GroupId string `xml:"groupId"` - ArtifactId string `xml:"artifactId"` - Versioning struct { - SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"` - } `xml:"versioning"` - Version string `xml:"version"` + GroupId string `xml:"groupId"` + ArtifactId string `xml:"artifactId"` + Versioning Versioning `xml:"versioning"` + Version string `xml:"version"` +} + +type Versioning struct { + SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"` } type SnapshotVersion struct { diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index c8fda40967bf..141ebbee0f7c 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -677,7 +677,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories") } -func (p *Parser) repoRequest(repo string, paths []string) (*http.Request, error) { +func (p *Parser) remoteRepoRequest(repo string, paths []string) (*http.Request, error) { repoURL, err := url.Parse(repo) if err != nil { p.logger.Error("URL parse error", log.String("repo", repo)) @@ -707,7 +707,7 @@ func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths` mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml") - req, err := p.repoRequest(repo, mavenMetadataPaths) + req, err := p.remoteRepoRequest(repo, mavenMetadataPaths) if err != nil { return "", xerrors.Errorf("unable to create request for maven-metadata.xml file") } @@ -722,13 +722,13 @@ func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) mavenMetadata, err := parseMavenMetadata(resp.Body) if err != nil { - return "", xerrors.Errorf("failed to parse the remote POM: %w", err) + return "", xerrors.Errorf("failed to parse maven-metadata.xml file: %w", err) } var pomFileName string for _, sv := range mavenMetadata.Versioning.SnapshotVersions { if sv.Extension == "pom" { - // mavenMetadataPaths[len(mavenMetadataPaths)-3] is artifactID + // mavenMetadataPaths[len(mavenMetadataPaths)-3] is always artifactID pomFileName = fmt.Sprintf("%s-%s.pom", mavenMetadataPaths[len(mavenMetadataPaths)-3], sv.Value) } } @@ -737,7 +737,7 @@ func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) } func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) { - req, err := p.repoRequest(repo, paths) + req, err := p.remoteRepoRequest(repo, paths) if err != nil { return nil, xerrors.Errorf("unable to create request for pom file") }