From d03f333cb7fe1b1b61fc1840e453109d444d4d0b Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Fri, 26 Apr 2024 11:43:55 +0200 Subject: [PATCH] Add support for fetching dependencies with Gradle Signed-off-by: Juan Manuel Leflet Estrada --- .../pkg/java_external_provider/dependency.go | 210 ++++++++++++++++ .../java_external_provider/dependency_test.go | 235 ++++++++++++++++++ 2 files changed, 445 insertions(+) diff --git a/external-providers/java-external-provider/pkg/java_external_provider/dependency.go b/external-providers/java-external-provider/pkg/java_external_provider/dependency.go index acaacc42..10ec6c23 100644 --- a/external-providers/java-external-provider/pkg/java_external_provider/dependency.go +++ b/external-providers/java-external-provider/pkg/java_external_provider/dependency.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "io" "io/fs" @@ -36,6 +37,21 @@ const ( baseDepKey = "baseDep" ) +const ( + maven = "maven" + gradle = "gradle" +) + +func (p *javaServiceClient) getBuildTool() string { + bf := "" + if bf = p.findPom(); bf != "" { + return maven + } else if bf = p.findGradleBuild(); bf != "" { + return gradle + } + return "" +} + // TODO implement this for real func (p *javaServiceClient) findPom() string { var depPath string @@ -51,10 +67,41 @@ func (p *javaServiceClient) findPom() string { if err != nil { return "" } + if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) { + return "" + } return f } +func (p *javaServiceClient) findGradleBuild() string { + // TODO: naive? + if p.config.Location != "" { + f, err := filepath.Abs(filepath.Join(p.config.Location, "build.gradle")) + if err != nil { + return "" + } + return f + } + return "" +} + func (p *javaServiceClient) GetDependencies(ctx context.Context) (map[uri.URI][]*provider.Dep, error) { + if p.getBuildTool() == gradle { + p.log.V(2).Info("gradle found - retrieving dependencies") + m := map[uri.URI][]*provider.Dep{} + deps, err := p.getDependenciesForGradle(ctx) + for f, ds := range deps { + deps := []*provider.Dep{} + for _, dep := range ds { + d := dep.Dep + deps = append(deps, &d) + deps = append(deps, provider.ConvertDagItemsToList(dep.AddedDeps)...) + } + m[f] = deps + } + return m, err + } + p.depsMutex.RLock() val := p.depsCache p.depsMutex.RUnlock() @@ -226,6 +273,17 @@ func pomCoordinate(value *string) string { } func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { + switch p.getBuildTool() { + case maven: + return p.getDependenciesForMaven(ctx) + case gradle: + return p.getDependenciesForGradle(ctx) + default: + return nil, fmt.Errorf("no build tool found") + } +} + +func (p *javaServiceClient) getDependenciesForMaven(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { localRepoPath := getMavenLocalRepoPath(p.mvnSettingsFile) path := p.findPom() @@ -274,6 +332,158 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI return m, nil } +// getDependenciesForGradle invokes the Gradle wrapper to get the dependency tree and returns all project dependencies +// TODO: what if no wrapper? +func (p *javaServiceClient) getDependenciesForGradle(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { + subprojects, err := p.getGradleSubprojects() + if err != nil { + return nil, err + } + + // command syntax: ./gradlew subproject1:dependencies subproject2:dependencies ... + args := []string{} + if len(subprojects) > 0 { + for _, sp := range subprojects { + args = append(args, fmt.Sprintf("%s:dependencies", sp)) + } + } else { + args = append(args, "dependencies") + } + + // get the graph output + cmd := exec.Command("./gradlew", args...) + cmd.Dir = p.config.Location + output, err := cmd.CombinedOutput() + if err != nil { + return nil, err + } + + lines := strings.Split(string(output), "\n") + deps := p.parseGradleDependencyOutput(lines) + + // TODO: do we need to separate by submodule somehow? + + path := p.findGradleBuild() + file := uri.File(path) + m := map[uri.URI][]provider.DepDAGItem{} + m[file] = deps + + // TODO: need error? + return m, nil +} + +func (p *javaServiceClient) getGradleSubprojects() ([]string, error) { + args := []string{ + "projects", + } + + // Ideally we'd want to set this in gradle.properties, or as a -Dorg.gradle.java.home arg, + // but it doesn't seem to work in older Gradle versions. This should only affect child processes in any case. + err := os.Setenv("JAVA_HOME", os.Getenv("JAVA8_HOME")) + if err != nil { + return nil, err + } + + exe := filepath.Join(p.config.Location, "gradlew") + cmd := exec.Command(exe, args...) + cmd.Dir = p.config.Location + output, err := cmd.CombinedOutput() + if err != nil { + return nil, err + } + + beginRegex := regexp.MustCompile(`Root project`) + endRegex := regexp.MustCompile(`To see a list of`) + npRegex := regexp.MustCompile(`No sub-projects`) + pRegex := regexp.MustCompile(`.*- Project '(.*)'`) + + subprojects := []string{} + + gather := false + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if npRegex.Find([]byte(line)) != nil { + return []string{}, nil + } + if beginRegex.Find([]byte(line)) != nil { + gather = true + continue + } + if gather { + if endRegex.Find([]byte(line)) != nil { + return subprojects, nil + } + + if p := pRegex.FindStringSubmatch(line); p != nil { + subprojects = append(subprojects, p[1]) + } + } + } + + return subprojects, fmt.Errorf("error parsing gradle dependency output") +} + +// parseGradleDependencyOutput converts the relevant lines from the dependency output into actual dependencies +// See https://regex101.com/r/9Gp7dW/1 for context +func (p *javaServiceClient) parseGradleDependencyOutput(lines []string) []provider.DepDAGItem { + deps := []provider.DepDAGItem{} + + treeDepRegex := regexp.MustCompile(`^([| ]+)?[+\\]--- (.*)`) + + // map of to + // this is so that children can be added to their respective parents + lastFoundWithDepth := make(map[int]*provider.DepDAGItem) + + for _, line := range lines { + match := treeDepRegex.FindStringSubmatch(line) + if match != nil { + dep := parseGradleDependencyString(match[2]) + if reflect.DeepEqual(dep, provider.DepDAGItem{}) { // ignore empty dependency + continue + } else if match[1] != "" { // transitive dependency + dep.Dep.Indirect = true + depth := len(match[1]) / 5 // get the level of anidation of the dependency within the tree + parent := lastFoundWithDepth[depth-1] // find its parent + parent.AddedDeps = append(parent.AddedDeps, dep) // add child to parent + lastFoundWithDepth[depth] = &parent.AddedDeps[len(parent.AddedDeps)-1] // update last found with given depth + } else { // root level (direct) dependency + deps = append(deps, dep) // add root dependency to result list + lastFoundWithDepth[0] = &deps[len(deps)-1] + continue + } + } + } + + return deps +} + +// parseGradleDependencyString parses the lines of the gradle dependency output, for instance: +// org.codehaus.groovy:groovy:3.0.21 +// org.codehaus.groovy:groovy:3.+ -> 3.0.21 +// com.codevineyard:hello-world:{strictly 1.0.1} -> 1.0.1 +// :simple-jar (n) +func parseGradleDependencyString(s string) provider.DepDAGItem { + // (*) - dependencies omitted (listed previously) + // (n) - Not resolved (configuration is not meant to be resolved) + if strings.HasSuffix(s, "(n)") || strings.HasSuffix(s, "(*)") { + return provider.DepDAGItem{} + } + + depRegex := regexp.MustCompile(`(.+):(.+):((.*) -> )?(.*)`) + libRegex := regexp.MustCompile(`:(.*)`) + + dep := provider.Dep{} + match := depRegex.FindStringSubmatch(s) + if match != nil { + dep.Name = match[1] + "." + match[2] + dep.Version = match[5] + } else if match = libRegex.FindStringSubmatch(s); match != nil { + dep.Name = match[1] + } + + return provider.DepDAGItem{Dep: dep, AddedDeps: []provider.DepDAGItem{}} +} + // extractSubmoduleTrees creates an array of lines for each submodule tree found in the mvn dependency:tree output func extractSubmoduleTrees(lines []string) [][]string { submoduleTrees := [][]string{} diff --git a/external-providers/java-external-provider/pkg/java_external_provider/dependency_test.go b/external-providers/java-external-provider/pkg/java_external_provider/dependency_test.go index d605f319..33957093 100644 --- a/external-providers/java-external-provider/pkg/java_external_provider/dependency_test.go +++ b/external-providers/java-external-provider/pkg/java_external_provider/dependency_test.go @@ -560,3 +560,238 @@ func Test_parseMavenDepLines(t *testing.T) { }) } } + +func Test_parseGradleDependencyOutput(t *testing.T) { + gradleOutput := ` +Starting a Gradle Daemon, 1 incompatible Daemon could not be reused, use --status for details + +> Task :dependencies + +------------------------------------------------------------ +Root project +------------------------------------------------------------ + +annotationProcessor - Annotation processors and their dependencies for source set 'main'. +No dependencies + +api - API dependencies for source set 'main'. (n) +No dependencies + +apiElements - API elements for main. (n) +No dependencies + +archives - Configuration for archive artifacts. (n) +No dependencies + +compileClasspath - Compile classpath for source set 'main'. ++--- org.codehaus.groovy:groovy:3.+ -> 3.0.21 ++--- org.codehaus.groovy:groovy-json:3.+ -> 3.0.21 +| \--- org.codehaus.groovy:groovy:3.0.21 ++--- com.codevineyard:hello-world:{strictly 1.0.1} -> 1.0.1 +\--- :simple-jar + +testRuntimeOnly - Runtime only dependencies for source set 'test'. (n) +No dependencies + +(*) - dependencies omitted (listed previously) + +(n) - Not resolved (configuration is not meant to be resolved) + +A web-based, searchable dependency report is available by adding the --scan option. + +BUILD SUCCESSFUL in 4s +1 actionable task: 1 executed +` + + lines := strings.Split(gradleOutput, "\n") + + p := javaServiceClient{ + log: testr.New(t), + depToLabels: map[string]*depLabelItem{}, + config: provider.InitConfig{ + ProviderSpecificConfig: map[string]interface{}{ + "excludePackages": []string{}, + }, + }, + } + + wantedDeps := []provider.DepDAGItem{ + { + Dep: provider.Dep{ + Name: "org.codehaus.groovy.groovy", + Version: "3.0.21", + Indirect: false, + }, + }, + { + Dep: provider.Dep{ + Name: "org.codehaus.groovy.groovy-json", + Version: "3.0.21", + Indirect: false, + }, + AddedDeps: []provider.DepDAGItem{ + { + Dep: provider.Dep{ + Name: "org.codehaus.groovy.groovy", + Version: "3.0.21", + Indirect: true, + }, + }, + }, + }, + { + Dep: provider.Dep{ + Name: "com.codevineyard.hello-world", + Version: "1.0.1", + Indirect: false, + }, + }, + { + Dep: provider.Dep{ + Name: "simple-jar", + Indirect: false, + }, + }, + } + + deps := p.parseGradleDependencyOutput(lines) + + if len(deps) != len(wantedDeps) { + t.Errorf("different number of dependencies found") + } + + for i := 0; i < len(deps); i++ { + dep := deps[i] + wantedDep := wantedDeps[i] + if dep.Dep.Name != wantedDep.Dep.Name { + t.Errorf("wanted name: %s, found name: %s", wantedDep.Dep.Name, dep.Dep.Name) + } + if dep.Dep.Version != wantedDep.Dep.Version { + t.Errorf("wanted version: %s, found version: %s", wantedDep.Dep.Version, dep.Dep.Version) + } + if len(dep.AddedDeps) != len(wantedDep.AddedDeps) { + t.Errorf("wanted %d child deps, found %d for dep %s", len(wantedDep.AddedDeps), len(dep.AddedDeps), dep.Dep.Name) + } + + } + +} + +func Test_parseGradleDependencyOutput_withTwoLevelsOfNesting(t *testing.T) { + gradleOutput := ` +Starting a Gradle Daemon, 1 incompatible Daemon could not be reused, use --status for details + +> Task :dependencies + +------------------------------------------------------------ +Root project +------------------------------------------------------------ + +annotationProcessor - Annotation processors and their dependencies for source set 'main'. +No dependencies + +api - API dependencies for source set 'main'. (n) +No dependencies + +apiElements - API elements for main. (n) +No dependencies + +archives - Configuration for archive artifacts. (n) +No dependencies + +compileClasspath - Compile classpath for source set 'main'. ++--- net.sourceforge.pmd:pmd-java:5.6.1 + +--- net.sourceforge.pmd:pmd-core:5.6.1 + | \--- com.google.code.gson:gson:2.5 + \--- net.sourceforge.saxon:saxon:9.1.0.8 ++--- org.apache.logging.log4j:log4j-api:2.9.1 + +testRuntimeOnly - Runtime only dependencies for source set 'test'. (n) +No dependencies + +(*) - dependencies omitted (listed previously) + +(n) - Not resolved (configuration is not meant to be resolved) + +A web-based, searchable dependency report is available by adding the --scan option. + +BUILD SUCCESSFUL in 4s +1 actionable task: 1 executed +` + + lines := strings.Split(gradleOutput, "\n") + + p := javaServiceClient{ + log: testr.New(t), + depToLabels: map[string]*depLabelItem{}, + config: provider.InitConfig{ + ProviderSpecificConfig: map[string]interface{}{ + "excludePackages": []string{}, + }, + }, + } + + wantedDeps := []provider.DepDAGItem{ + { + Dep: provider.Dep{ + Name: "net.sourceforge.pmd.pmd-java", + Version: "5.6.1", + Indirect: false, + }, + AddedDeps: []provider.DepDAGItem{ + { + Dep: provider.Dep{ + Name: "net.sourceforge.pmd.pmd-core", + Version: "5.6.1", + Indirect: true, + }, + AddedDeps: []provider.DepDAGItem{ + { + Dep: provider.Dep{ + Name: "com.google.code.gson.gson", + Version: "2.5", + Indirect: true, + }, + }, + }, + }, + { + Dep: provider.Dep{ + Name: "net.sourceforge.saxon.saxon", + Version: "9.1.0.8", + Indirect: true, + }, + }, + }, + }, + { + Dep: provider.Dep{ + Name: "org.apache.logging.log4j.log4j-api", + Version: "2.9.1", + Indirect: false, + }, + }, + } + + deps := p.parseGradleDependencyOutput(lines) + + if len(deps) != len(wantedDeps) { + t.Errorf("different number of dependencies found") + } + + for i := 0; i < len(deps); i++ { + dep := deps[i] + wantedDep := wantedDeps[i] + if dep.Dep.Name != wantedDep.Dep.Name { + t.Errorf("wanted name: %s, found name: %s", wantedDep.Dep.Name, dep.Dep.Name) + } + if dep.Dep.Version != wantedDep.Dep.Version { + t.Errorf("wanted version: %s, found version: %s", wantedDep.Dep.Version, dep.Dep.Version) + } + if len(dep.AddedDeps) != len(wantedDep.AddedDeps) { + t.Errorf("wanted %d child deps, found %d for dep %s", len(wantedDep.AddedDeps), len(dep.AddedDeps), dep.Dep.Name) + } + + } + +}