Use gradle-dep-tree with Audit (#719)

artifactory/commands/gradle/gradle.go

@@ -56,7 +56,7 @@ func (gc *GradleCommand) SetServerDetails(serverDetails *config.ServerDetails) *
 
 func (gc *GradleCommand) init() (vConfig *viper.Viper, err error) {
  // Read config
- vConfig, err = utils.ReadGradleConfig(gc.configPath, nil)
+ vConfig, err = utils.ReadConfigFile(gc.configPath, utils.YAML)
  if err != nil {
  return
  }

artifactory/utils/buildinfoproperties.go

@@ -155,15 +155,6 @@ func ReadConfigFile(configPath string, configType ConfigType) (config *viper.Vip
  return config, errorutils.CheckError(err)
 }
 
-func ReadGradleConfig(path string, gradleConfigParams map[string]any) (config *viper.Viper, err error) {
- if path == "" {
- config = createDefaultConfigWithParams(YAML, Gradle.String(), gradleConfigParams)
- } else {
- config, err = ReadConfigFile(path, YAML)
- }
- return
-}
-
 func ReadMavenConfig(path string, mvnProps map[string]any) (config *viper.Viper, err error) {
  if path == "" {
  config = createDefaultConfigWithParams(YAML, Maven.String(), mvnProps)

utils/config/config.go

@@ -589,7 +589,7 @@ type MissionControlDetails struct {
 }
 
 func (serverDetails *ServerDetails) IsEmpty() bool {
- return len(serverDetails.ServerId) == 0
+ return len(serverDetails.ServerId) == 0 && serverDetails.Url == ""
 }
 
 func (serverDetails *ServerDetails) SetUser(username string) {

utils/coreutils/techutils.go

@@ -58,9 +58,9 @@ var technologiesData = map[Technology]TechData{
  execCommand: "mvn",
  },
  Gradle: {
- indicators: []string{".gradle"},
+ indicators: []string{".gradle", ".gradle.kts"},
  ciSetupSupport: true,
- packageDescriptor: "build.gradle",
+ packageDescriptor: "build.gradle, build.gradle.kts",
  },
  Npm: {
  indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"},

utils/mvn/utils.go

@@ -14,7 +14,8 @@ import (
  "github.com/spf13/viper"
 )
 
-func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration, goals []string, threads int, insecureTls, disableDeploy bool) error {
+func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration,
+ goals []string, threads int, insecureTls, disableDeploy bool) error {
  buildInfoService := utils.CreateBuildInfoService()
  buildName, err := buildConf.GetBuildName()
  if err != nil {

xray/audit/java/gradle.go

@@ -1,70 +1,269 @@
 package java
 
 import (
+ "encoding/base64"
+ "encoding/json"
  "fmt"
+ "github.com/jfrog/build-info-go/build"
  "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
+ "github.com/jfrog/jfrog-cli-core/v2/utils/config"
  "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
- gradleutils "github.com/jfrog/jfrog-cli-core/v2/utils/gradle"
+ "github.com/jfrog/jfrog-client-go/utils/errorutils"
  "github.com/jfrog/jfrog-client-go/utils/io/fileutils"
  "github.com/jfrog/jfrog-client-go/utils/log"
  "github.com/jfrog/jfrog-client-go/xray/services"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
 )
 
-const gradlew = "gradlew"
+const (
+ remoteDepTreePath = "artifactory/oss-releases"
+ gradlew = "gradlew"
+ depTreeInitFile = "gradledeptree.init"
+ depTreeOutputFile = "gradledeptree.out"
+ depTreeInitScript = `initscript {
+ repositories { %s
+ mavenCentral()
+ }
+ dependencies {
+ classpath 'com.jfrog:gradle-dep-tree:2.2.0'
+ }
+}
+
+allprojects {
+ repositories { %s
+ }
+ apply plugin: com.jfrog.GradleDepTree
+}`
+ artifactoryRepository = `
+ maven {
+ url "%s/%s"
+ credentials {
+ username = '%s'
+ password = '%s'
+ }
+ }`
+)
+
+type depTreeManager struct {
+ dependenciesTree
+ server *config.ServerDetails
+ releasesRepo string
+ depsRepo string
+ useWrapper bool
+}
+
+// dependenciesTree represents a map between dependencies to their children dependencies in multiple projects.
+type dependenciesTree struct {
+ tree map[string][]dependenciesPaths
+}
+
+// dependenciesPaths represents a map between dependencies to their children dependencies in a single project.
+type dependenciesPaths struct {
+ Paths map[string]dependenciesPaths `json:"children"`
+}
+
+// The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project.
+// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in dependenciesTree struct.
+func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error {
+ outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n")
+ for _, path := range outputFiles {
+ tree, err := os.ReadFile(strings.TrimSpace(path))
+ if err != nil {
+ return errorutils.CheckError(err)
+ }
+
+ encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:]
+ decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName)
+ if err != nil {
+ return errorutils.CheckError(err)
+ }
+
+ if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil {
+ return errorutils.CheckError(err)
+ }
+ }
+ return nil
+}
 
-func buildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (dependencyTree []*services.GraphNode, err error) {
- buildConfiguration, cleanBuild := createBuildConfiguration("audit-gradle")
+func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error {
+ var deps dependenciesPaths
+ if err := json.Unmarshal(jsonDepTree, &deps); err != nil {
+ return errorutils.CheckError(err)
+ }
+ if dtp.tree == nil {
+ dtp.tree = make(map[string][]dependenciesPaths)
+ }
+ dtp.tree[fileName] = append(dtp.tree[fileName], deps)
+ return nil
+}
+
+func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, depsRepo, releasesRepo string) (dependencyTree []*services.GraphNode, err error) {
+ if (server != nil && server.IsEmpty()) || depsRepo == "" {
+ depsRepo, server, err = getGradleConfig()
+ if err != nil {
+ return
+ }
+ }
+
+ manager := &depTreeManager{
+ server: server,
+ releasesRepo: releasesRepo,
+ depsRepo: depsRepo,
+ useWrapper: useWrapper,
+ }
+
+ outputFileContent, err := manager.runGradleDepTree()
+ if err != nil {
+ return nil, err
+ }
+ return manager.getGraphFromDepTree(outputFileContent)
+}
+
+func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err error) {
+ // Create the script file in the repository
+ depTreeDir, err := dtp.createDepTreeScript()
+ if err != nil {
+ return
+ }
  defer func() {
- e := cleanBuild()
+ e := fileutils.RemoveTempDir(depTreeDir)
  if err == nil {
  err = e
  }
  }()
 
- err = runGradle(buildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile, gradleConfigParams)
+ if dtp.useWrapper {
+ dtp.useWrapper, err = isGradleWrapperExist()
+ if err != nil {
+ return
+ }
+ }
+
+ return dtp.execGradleDepTree(depTreeDir)
+}
+
+func (dtp *depTreeManager) createDepTreeScript() (tmpDir string, err error) {
+ tmpDir, err = fileutils.CreateTempDir()
+ if err != nil {
+ return
+ }
+ depsRepo := ""
+ releasesRepo := ""
+ if dtp.server != nil {
+ releasesRepo, err = getDepTreeArtifactoryRepository(fmt.Sprintf("%s/%s", dtp.releasesRepo, remoteDepTreePath), dtp.server)
+ if err != nil {
+ return
+ }
+ depsRepo, err = getDepTreeArtifactoryRepository(dtp.depsRepo, dtp.server)
+ if err != nil {
+ return
+ }
+ }
+ depTreeInitScript := fmt.Sprintf(depTreeInitScript, releasesRepo, depsRepo)
+ return tmpDir, errorutils.CheckError(os.WriteFile(filepath.Join(tmpDir, depTreeInitFile), []byte(depTreeInitScript), 0666))
+}
+
+func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileContent []byte, err error) {
+ gradleExecPath, err := build.GetGradleExecPath(dtp.useWrapper)
  if err != nil {
+ err = errorutils.CheckError(err)
  return
  }
 
- dependencyTree, err = createGavDependencyTree(buildConfiguration)
+ outputFilePath := filepath.Join(depTreeDir, depTreeOutputFile)
+ tasks := []string{
+ "clean",
+ "generateDepTrees", "-I", filepath.Join(depTreeDir, depTreeInitFile),
+ "-q",
+ fmt.Sprintf("-Dcom.jfrog.depsTreeOutputFile=%s", outputFilePath),
+ "-Dcom.jfrog.includeAllBuildFiles=true"}
+ log.Info("Running gradle dep tree command: ", gradleExecPath, tasks)
+ if output, err := exec.Command(gradleExecPath, tasks...).CombinedOutput(); err != nil {
+ return nil, errorutils.CheckErrorf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output))
+ }
+ defer func() {
+ e := errorutils.CheckError(os.Remove(outputFilePath))
+ if err == nil {
+ err = e
+ }
+ }()
+
+ outputFileContent, err = os.ReadFile(outputFilePath)
+ err = errorutils.CheckError(err)
  return
 }
 
-func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (err error) {
- tasks := "clean compileJava "
- if !excludeTestDeps {
- tasks += "compileTestJava "
- }
- tasks += "artifactoryPublish"
- log.Debug(fmt.Sprintf("gradle command tasks: %v", tasks))
- configFilePath := ""
- if !ignoreConfigFile {
- var exists bool
- configFilePath, exists, err = utils.GetProjectConfFilePath(utils.Gradle)
- if err != nil {
- return
+// Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input
+func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*services.GraphNode, error) {
+ if err := dtp.parseDepTreeFiles(outputFileContent); err != nil {
+ return nil, err
+ }
+ var depsGraph []*services.GraphNode
+ for dependency, children := range dtp.tree {
+ directDependency := &services.GraphNode{
+ Id: GavPackageTypeIdentifier + dependency,
+ Nodes: []*services.GraphNode{},
  }
- if exists {
- log.Debug("Using resolver config from", configFilePath)
+ for _, childPath := range children {
+ populateGradleDependencyTree(directDependency, childPath)
  }
+ depsGraph = append(depsGraph, directDependency)
  }
- // Check whether gradle wrapper exists
- if useWrapper {
- useWrapper, err = isGradleWrapperExist()
- if err != nil {
- return
+ return depsGraph, nil
+}
+
+func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren dependenciesPaths) {
+ for gav, children := range currNodeChildren.Paths {
+ childNode := &services.GraphNode{
+ Id: GavPackageTypeIdentifier + gav,
+ Nodes: []*services.GraphNode{},
+ Parent: currNode,
  }
- if gradleConfigParams == nil {
- gradleConfigParams = make(map[string]any)
+ if currNode.NodeHasLoop() {
+ return
  }
- gradleConfigParams["usewrapper"] = useWrapper
+ populateGradleDependencyTree(childNode, children)
+ currNode.Nodes = append(currNode.Nodes, childNode)
+ }
+}
+
+func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) {
+ pass := server.Password
+ user := server.User
+ if server.AccessToken != "" {
+ pass = server.AccessToken
  }
- // Read config
- vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams)
+ if pass == "" && user == "" {
+ return "", fmt.Errorf("either username/password or access token must be set for %s", server.Url)
+ }
+ return fmt.Sprintf(artifactoryRepository,
+ strings.TrimSuffix(server.ArtifactoryUrl, "/"),
+ remoteRepo,
+ user,
+ pass), nil
+}
+
+// getGradleConfig the remote repository and server details defined in the .jfrog/projects/gradle.yaml file, if configured.
+func getGradleConfig() (string, *config.ServerDetails, error) {
+ var exists bool
+ configFilePath, exists, err := utils.GetProjectConfFilePath(utils.Gradle)
+ if err != nil || !exists {
+ return "", nil, err
+ }
+ log.Debug("Using resolver config from", configFilePath)
+ configContent, err := utils.ReadConfigFile(configFilePath, utils.YAML)
  if err != nil {
- return err
+ return "", nil, err
+ }
+ var repository string
+ if configContent.IsSet("resolver.repo") {
+ repository = configContent.Get("resolver.repo").(string)
  }
- return gradleutils.RunGradle(vConfig, tasks, "", buildConfiguration, 0, true)
+ server, err := utils.GetServerDetails(configContent)
+ return repository, server, err
 }
 
 // This function assumes that the Gradle wrapper is in the root directory.