From 5e8666c620532c414c67d55b498fc6cb4247db9e Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Sun, 15 Jan 2023 16:22:32 +0200 Subject: [PATCH 01/48] first commit --- artifactory/utils/dependenciesutils.go | 4 +- utils/gradle/utils.go | 17 +- utils/mvn/utils.go | 17 +- xray/audit/java/gradle.go | 2 +- xray/commands/audit/generic/auditmanager.go | 224 ++++++++++++-------- xray/commands/audit/generic/generic.go | 29 ++- 6 files changed, 181 insertions(+), 112 deletions(-) diff --git a/artifactory/utils/dependenciesutils.go b/artifactory/utils/dependenciesutils.go index e6fc348b7..c5aade5ce 100644 --- a/artifactory/utils/dependenciesutils.go +++ b/artifactory/utils/dependenciesutils.go @@ -35,7 +35,7 @@ func DownloadExtractorIfNeeded(targetPath, downloadPath string) error { return err } - return downloadExtractor(artDetails, remotePath, targetPath) + return DownloadExtractor(artDetails, remotePath, targetPath) } func GetExtractorsRemoteDetails(downloadPath string) (*config.ServerDetails, string, error) { @@ -61,7 +61,7 @@ func getExtractorsRemoteDetails(extractorsRemote, downloadPath string) (*config. return serverDetails, path.Join(repoName, downloadPath), err } -func downloadExtractor(artDetails *config.ServerDetails, downloadPath, targetPath string) error { +func DownloadExtractor(artDetails *config.ServerDetails, downloadPath, targetPath string) error { downloadUrl := fmt.Sprintf("%s%s", artDetails.ArtifactoryUrl, downloadPath) log.Info("Downloading build-info-extractor from", downloadUrl) filename, localDir := fileutils.GetFileAndDirFromPath(targetPath) diff --git a/utils/gradle/utils.go b/utils/gradle/utils.go index 38c3159b4..7cc21cb8e 100644 --- a/utils/gradle/utils.go +++ b/utils/gradle/utils.go @@ -17,7 +17,7 @@ const ( useWrapper = "usewrapper" ) -func RunGradle(vConfig *viper.Viper, tasks, deployableArtifactsFile string, configuration *utils.BuildConfiguration, threads int, useWrapperIfMissingConfig, disableDeploy bool) error { +func RunGradle(vConfig *viper.Viper, tasks, deployableArtifactsFile string, configuration *utils.BuildConfiguration, threads int, disableDeploy bool) error { buildInfoService := utils.CreateBuildInfoService() buildName, err := configuration.GetBuildName() if err != nil { @@ -35,20 +35,27 @@ func RunGradle(vConfig *viper.Viper, tasks, deployableArtifactsFile string, conf if err != nil { return errorutils.CheckError(err) } - props, wrapper, plugin, err := createGradleRunConfig(vConfig, deployableArtifactsFile, configuration, threads, useWrapperIfMissingConfig, disableDeploy) + props, wrapper, plugin, err := createGradleRunConfig(vConfig, deployableArtifactsFile, threads, disableDeploy) if err != nil { return err } - dependenciesPath, err := config.GetJfrogDependenciesPath() + dependencyLocalPath, err := GetGradleDependencyLocalPath() if err != nil { return err } - dependencyLocalPath := filepath.Join(dependenciesPath, "gradle", build.GradleExtractorDependencyVersion) gradleModule.SetExtractorDetails(dependencyLocalPath, filepath.Join(coreutils.GetCliPersistentTempDirPath(), utils.PropertiesTempPath), strings.Split(tasks, " "), wrapper, plugin, utils.DownloadExtractorIfNeeded, props) return coreutils.ConvertExitCodeError(gradleModule.CalcDependencies()) } -func createGradleRunConfig(vConfig *viper.Viper, deployableArtifactsFile string, buildConf *utils.BuildConfiguration, threads int, useWrapperIfMissingConfig, disableDeploy bool) (props map[string]string, wrapper, plugin bool, err error) { +func GetGradleDependencyLocalPath() (string, error) { + dependenciesPath, err := config.GetJfrogDependenciesPath() + if err != nil { + return "", err + } + return filepath.Join(dependenciesPath, "gradle", build.GradleExtractorDependencyVersion), nil +} + +func createGradleRunConfig(vConfig *viper.Viper, deployableArtifactsFile string, threads int, disableDeploy bool) (props map[string]string, wrapper, plugin bool, err error) { wrapper = vConfig.GetBool(useWrapper) if threads > 0 { vConfig.Set(utils.ForkCount, threads) diff --git a/utils/mvn/utils.go b/utils/mvn/utils.go index a88c7011e..79c0a377d 100644 --- a/utils/mvn/utils.go +++ b/utils/mvn/utils.go @@ -32,10 +32,6 @@ func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *u if err != nil { return errorutils.CheckError(err) } - dependenciesPath, err := config.GetJfrogDependenciesPath() - if err != nil { - return err - } props, err := createMvnRunProps(vConfig, buildArtifactsDetailsFile, buildConf, goals, threads, insecureTls, disableDeploy) if err != nil { return err @@ -47,11 +43,22 @@ func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *u if v, ok := props["buildInfoConfig.artifactoryResolutionEnabled"]; ok { mvnOpts = append(mvnOpts, "-DbuildInfoConfig.artifactoryResolutionEnabled="+v) } - dependencyLocalPath := filepath.Join(dependenciesPath, "maven", build.MavenExtractorDependencyVersion) + dependencyLocalPath, err := GetMavenDependencyLocalPath() + if err != nil { + return err + } mavenModule.SetExtractorDetails(dependencyLocalPath, filepath.Join(coreutils.GetCliPersistentTempDirPath(), utils.PropertiesTempPath), goals, utils.DownloadExtractorIfNeeded, props).SetMavenOpts(mvnOpts...) return coreutils.ConvertExitCodeError(mavenModule.CalcDependencies()) } +func GetMavenDependencyLocalPath() (string, error) { + dependenciesPath, err := config.GetJfrogDependenciesPath() + if err != nil { + return "", err + } + return filepath.Join(dependenciesPath, "maven", build.MavenExtractorDependencyVersion), nil +} + func createMvnRunProps(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration, goals []string, threads int, insecureTls, disableDeploy bool) (map[string]string, error) { vConfig.Set(utils.InsecureTls, insecureTls) if threads > 0 { diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index fd9aa070c..b4721b79a 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -55,7 +55,7 @@ func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, us if err != nil { return err } - return gradleutils.RunGradle(vConfig, tasks, "", buildConfiguration, 0, useWrapper, true) + return gradleutils.RunGradle(vConfig, tasks, "", buildConfiguration, 0, true) } // This function assumes that the Gradle wrapper is in the root directory. diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 5ceab378e..9155df0f4 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -23,133 +23,191 @@ import ( "github.com/jfrog/jfrog-client-go/xray/services" ) +type Params struct { + xrayGraphScanParams services.XrayGraphScanParams + serverDetails *config.ServerDetails + progress ioUtils.ProgressMgr + ignoreConfigFile bool + excludeTestDeps bool + insecureTls bool + useWrapper bool + requirementsFile string + technologies []string + workingDirs []string + args []string +} + +func NewAuditParams() *Params { + return &Params{} +} + +func (params *Params) SetXrayGraphScanParams(xrayGraphScanParams services.XrayGraphScanParams) *Params { + params.xrayGraphScanParams = xrayGraphScanParams + return params +} + +func (params *Params) SetServerDetails(serverDetails *config.ServerDetails) *Params { + params.serverDetails = serverDetails + return params +} + +func (params *Params) SetExcludeTestDeps(excludeTestDeps bool) *Params { + params.excludeTestDeps = excludeTestDeps + return params +} + +func (params *Params) SetUseWrapper(useWrapper bool) *Params { + params.useWrapper = useWrapper + return params +} + +func (params *Params) SetInsecureTLS(insecureTls bool) *Params { + params.insecureTls = insecureTls + return params +} + +func (params *Params) SetArgs(args []string) *Params { + params.args = args + return params +} + +func (params *Params) SetProgressBar(progress ioUtils.ProgressMgr) *Params { + params.progress = progress + return params +} + +func (params *Params) SetRequirementsFile(requirementsFile string) *Params { + params.requirementsFile = requirementsFile + return params +} + +func (params *Params) SetIgnoreConfigFile(ignoreConfigFile bool) *Params { + params.ignoreConfigFile = ignoreConfigFile + return params +} + +func (params *Params) SetWorkingDirs(workingDirs []string) *Params { + params.workingDirs = workingDirs + return params +} + +func (params *Params) SetTechnologies(technologies ...string) *Params { + params.technologies = append(params.technologies, technologies...) + return params +} + // GenericAudit audits all the projects found in the given workingDirs -func GenericAudit( - xrayGraphScanParams services.XrayGraphScanParams, - serverDetails *config.ServerDetails, - excludeTestDeps, - useWrapper, - insecureTls bool, - args []string, - progress ioUtils.ProgressMgr, - requirementsFile string, - ignoreConfigFile bool, - workingDirs []string, - technologies ...string) (results []services.ScanResponse, isMultipleRoot bool, err error) { - - if len(workingDirs) == 0 { +func GenericAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bool, err error) { + if len(params.workingDirs) == 0 { log.Info("Auditing project: ") - return doAudit(xrayGraphScanParams, serverDetails, excludeTestDeps, useWrapper, insecureTls, args, progress, requirementsFile, ignoreConfigFile, technologies...) + return doAudit(params) } + + return auditMultipleWorkingDirs(params) +} + +func auditMultipleWorkingDirs(params *Params) (results []services.ScanResponse, isMultipleRoot bool, err error) { projectDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return } - var errorList []string defer func() { e := os.Chdir(projectDir) if err == nil { err = e } }() - for _, wd := range workingDirs { + var errorList strings.Builder + for _, wd := range params.workingDirs { absWd, e := filepath.Abs(wd) if e != nil { - // Save the error but continue to the other paths - errorList = append(errorList, fmt.Sprintf("the audit command couldn't find the following path: %s\n%s", wd, e.Error())) + errorList.WriteString(fmt.Sprintf("the audit command couldn't find the following path: %s\n%s\n", wd, e.Error())) continue } - log.Info("Auditing project: " + absWd) + log.Info("Auditing project:", absWd) e = os.Chdir(absWd) if e != nil { - // Save the error but continue to the other paths - errorList = append(errorList, fmt.Sprintf("the audit command couldn't change the current working directory to the following path: %s\n%s", absWd, e.Error())) + errorList.WriteString(fmt.Sprintf("the audit command couldn't change the current working directory to the following path: %s\n%s\n", absWd, e.Error())) continue } - techResults, isMultipleRootProject, e := doAudit(xrayGraphScanParams, serverDetails, excludeTestDeps, useWrapper, insecureTls, args, progress, requirementsFile, ignoreConfigFile, technologies...) + + techResults, isMultipleRootProject, e := doAudit(params) if e != nil { - // Save the error but continue to the other paths - errorList = append(errorList, fmt.Sprintf("audit command in %s failed:\n%s", absWd, e.Error())) - } else { - results = append(results, techResults...) - isMultipleRoot = isMultipleRootProject + errorList.WriteString(fmt.Sprintf("audit command in %s failed:\n%s\n", absWd, e.Error())) + continue } + + results = append(results, techResults...) + isMultipleRoot = isMultipleRootProject } - if len(errorList) > 0 { - err = errors.New(strings.Join(errorList, "\n")) + + if errorList.Len() > 0 { + err = errors.New(errorList.String()) } + return } -// Audits the project found in the current directory using Xray. -func doAudit( - xrayGraphScanParams services.XrayGraphScanParams, - serverDetails *config.ServerDetails, - excludeTestDeps, - useWrapper, - insecureTls bool, - args []string, - progress ioUtils.ProgressMgr, - requirementsFile string, - ignoreConfigFile bool, - technologies ...string) (results []services.ScanResponse, isMultipleRoot bool, err error) { - +// Audits the project found in the current directory using Xray. +func doAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bool, err error) { // If no technologies were given, try to detect all types of technologies used. // Otherwise, run audit for requested technologies only. - if len(technologies) == 0 { - technologies, err = detectedTechnologies() + if len(params.technologies) == 0 { + params.technologies, err = detectedTechnologies() if err != nil { return } } - var errorList []string - for _, tech := range coreutils.ToTechnologies(technologies) { - var dependencyTrees []*services.GraphNode - var e error - if progress != nil { - progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) - } - switch tech { - case coreutils.Maven: - dependencyTrees, e = java.BuildMvnDependencyTree(insecureTls, ignoreConfigFile) - case coreutils.Gradle: - dependencyTrees, e = java.BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile) - case coreutils.Npm: - dependencyTrees, e = npm.BuildDependencyTree(args) - case coreutils.Yarn: - dependencyTrees, e = yarn.BuildDependencyTree() - case coreutils.Go: - dependencyTrees, e = _go.BuildDependencyTree() - case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: - dependencyTrees, e = python.BuildDependencyTree(pythonutils.PythonTool(tech), requirementsFile) - case coreutils.Dotnet: + var errorList strings.Builder + for _, tech := range coreutils.ToTechnologies(params.technologies) { + if tech == coreutils.Dotnet { continue - case coreutils.Nuget: - dependencyTrees, e = nuget.BuildDependencyTree() - default: - e = errors.New(string(tech) + " is currently not supported") } - - var techResults []services.ScanResponse - if e == nil { - // If building the dependency tree was successful, run Xray scan. - techResults, e = audit.Audit(dependencyTrees, xrayGraphScanParams, serverDetails, progress, tech) + dependencyTrees, e := getTechDependencyTree(params, tech) + if e != nil { + errorList.WriteString(fmt.Sprintf("'%s' audit failed when building dependency tree:\n%s\n", tech, e.Error())) + continue } - + techResults, e := audit.Audit(dependencyTrees, params.xrayGraphScanParams, params.serverDetails, params.progress, tech) if e != nil { - // Save the error but continue to audit the next tech - errorList = append(errorList, fmt.Sprintf("'%s' audit command failed:\n%s", tech, e.Error())) - } else { - results = append(results, techResults...) - isMultipleRoot = len(dependencyTrees) > 1 + errorList.WriteString(fmt.Sprintf("'%s' audit command failed:\n%s\n", tech, e.Error())) + continue } + results = append(results, techResults...) + isMultipleRoot = len(dependencyTrees) > 1 } - if len(errorList) > 0 { - err = errors.New(strings.Join(errorList, "\n")) + if errorList.Len() > 0 { + err = errors.New(errorList.String()) } return } +func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependencyTrees []*services.GraphNode, e error) { + if params.progress != nil { + params.progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) + } + switch tech { + case coreutils.Maven: + dependencyTrees, e = java.BuildMvnDependencyTree(params.insecureTls, params.ignoreConfigFile) + case coreutils.Gradle: + dependencyTrees, e = java.BuildGradleDependencyTree(params.excludeTestDeps, params.useWrapper, params.ignoreConfigFile) + case coreutils.Npm: + dependencyTrees, e = npm.BuildDependencyTree(params.args) + case coreutils.Yarn: + dependencyTrees, e = yarn.BuildDependencyTree() + case coreutils.Go: + dependencyTrees, e = _go.BuildDependencyTree() + case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: + dependencyTrees, e = python.BuildDependencyTree(pythonutils.PythonTool(tech), params.requirementsFile) + case coreutils.Nuget: + dependencyTrees, e = nuget.BuildDependencyTree() + default: + e = errors.New(string(tech) + " is currently not supported") + } + + return dependencyTrees, e +} + func detectedTechnologies() (technologies []string, err error) { wd, err := os.Getwd() if errorutils.CheckError(err) != nil { diff --git a/xray/commands/audit/generic/generic.go b/xray/commands/audit/generic/generic.go index c6342e821..536395cdb 100644 --- a/xray/commands/audit/generic/generic.go +++ b/xray/commands/audit/generic/generic.go @@ -110,19 +110,18 @@ func (auditCmd *GenericAuditCommand) Run() (err error) { if err != nil { return } - results, isMultipleRootProject, auditErr := GenericAudit( - auditCmd.CreateXrayGraphScanParams(), - server, - auditCmd.excludeTestDependencies, - auditCmd.useWrapper, - auditCmd.insecureTls, - auditCmd.args, - auditCmd.progress, - auditCmd.requirementsFile, - false, - auditCmd.workingDirs, - auditCmd.technologies..., - ) + auditParams := NewAuditParams(). + SetXrayGraphScanParams(auditCmd.CreateXrayGraphScanParams()). + SetServerDetails(server). + SetExcludeTestDeps(auditCmd.excludeTestDependencies). + SetUseWrapper(auditCmd.useWrapper). + SetInsecureTLS(auditCmd.insecureTls). + SetArgs(auditCmd.args). + SetProgressBar(auditCmd.progress). + SetRequirementsFile(auditCmd.requirementsFile). + SetWorkingDirs(auditCmd.workingDirs). + SetTechnologies(auditCmd.technologies...) + results, isMultipleRootProject, auditErr := GenericAudit(auditParams) if auditCmd.progress != nil { err = auditCmd.progress.Quit() @@ -172,9 +171,7 @@ func (auditCmd *GenericAuditCommand) SetNpmScope(depType string) *GenericAuditCo } func (auditCmd *GenericAuditCommand) SetPipRequirementsFile(requirementsFile string) *GenericAuditCommand { - if requirementsFile != "" { - auditCmd.requirementsFile = requirementsFile - } + auditCmd.requirementsFile = requirementsFile return auditCmd } From 1094d3a5dae3afd8acef13c11add0b00cb2c25e2 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 13 Feb 2023 15:46:25 +0200 Subject: [PATCH 02/48] Improve deps resolution and refactor audit --- artifactory/commands/dotnet/dotnetcommand.go | 30 +-- artifactory/commands/golang/go.go | 19 +- .../commands/golang/testdata/.gitignore | 2 +- artifactory/commands/gradle/gradle.go | 4 +- artifactory/commands/mvn/mvn.go | 2 +- artifactory/commands/npm/common.go | 13 +- artifactory/commands/npm/generic.go | 2 +- artifactory/commands/npm/installorci.go | 13 +- artifactory/commands/python/poetry.go | 32 ++-- artifactory/commands/python/python.go | 70 ++++--- artifactory/commands/yarn/yarn.go | 174 +++++++++--------- artifactory/utils/buildinfoproperties.go | 25 +-- utils/mvn/utils.go | 3 +- xray/audit/go/golang.go | 18 +- xray/audit/java/gradle.go | 9 +- xray/audit/java/javautils.go | 17 ++ xray/audit/java/mvn.go | 8 +- xray/audit/java/mvn_test.go | 2 +- xray/audit/python/python.go | 149 ++++++++++----- xray/commands/audit/generic/auditmanager.go | 114 +++++++++++- .../testdata/poetry-project/pyproject.toml | 6 +- 21 files changed, 468 insertions(+), 244 deletions(-) diff --git a/artifactory/commands/dotnet/dotnetcommand.go b/artifactory/commands/dotnet/dotnetcommand.go index d18ea34b4..55e5f1a3e 100644 --- a/artifactory/commands/dotnet/dotnetcommand.go +++ b/artifactory/commands/dotnet/dotnetcommand.go @@ -219,7 +219,7 @@ func (dc *DotnetCommand) prepareConfigFile() (cleanup func() error, err error) { return fileutils.RemoveTempDir(tempDirPath) } - configFile, err := dc.InitNewConfig(tempDirPath) + configFile, err := InitNewConfig(tempDirPath, dc.repoName, dc.serverDetails, dc.useNugetV2) if err == nil { dc.argAndFlags = append(dc.argAndFlags, dc.GetToolchain().GetTypeFlagPrefix()+"configfile", configFile.Name()) } @@ -242,7 +242,7 @@ func getFlagValueIfExists(cmdFlag string, argAndFlags []string) (string, error) } // The fact that we here, means that neither of the flags were provided, and we need to init our own config. -func (dc *DotnetCommand) InitNewConfig(configDirPath string) (configFile *os.File, err error) { +func InitNewConfig(configDirPath, repoName string, server *config.ServerDetails, useNugetV2 bool) (configFile *os.File, err error) { // Initializing a new NuGet config file that NuGet will use into a temp file configFile, err = os.CreateTemp(configDirPath, configFilePattern) if errorutils.CheckError(err) != nil { @@ -259,20 +259,20 @@ func (dc *DotnetCommand) InitNewConfig(configDirPath string) (configFile *os.Fil // We would prefer to write the NuGet configuration using the `nuget add source` command, // but the NuGet configuration utility doesn't currently allow setting protocolVersion. // Until that is supported, the templated method must be used. - err = dc.addSourceToNugetTemplate(configFile) + err = addSourceToNugetTemplate(configFile, server, useNugetV2, repoName) return } // Adds a source to the nuget config template -func (dc *DotnetCommand) addSourceToNugetTemplate(configFile *os.File) error { - sourceUrl, user, password, err := dc.getSourceDetails() +func addSourceToNugetTemplate(configFile *os.File, server *config.ServerDetails, useNugetV2 bool, repoName string) error { + sourceUrl, user, password, err := GetSourceDetails(server, repoName, useNugetV2) if err != nil { return err } // Specify the protocolVersion protoVer := "3" - if dc.useNugetV2 { + if useNugetV2 { protoVer = "2" } @@ -281,28 +281,28 @@ func (dc *DotnetCommand) addSourceToNugetTemplate(configFile *os.File) error { return err } -func (dc *DotnetCommand) getSourceDetails() (sourceURL, user, password string, err error) { +func GetSourceDetails(details *config.ServerDetails, repoName string, useNugetV2 bool) (sourceURL, user, password string, err error) { var u *url.URL - u, err = url.Parse(dc.serverDetails.ArtifactoryUrl) + u, err = url.Parse(details.ArtifactoryUrl) if errorutils.CheckError(err) != nil { return } nugetApi := "api/nuget/v3" - if dc.useNugetV2 { + if useNugetV2 { nugetApi = "api/nuget" } - u.Path = path.Join(u.Path, nugetApi, dc.repoName) + u.Path = path.Join(u.Path, nugetApi, repoName) sourceURL = u.String() - user = dc.serverDetails.User - password = dc.serverDetails.Password + user = details.User + password = details.Password // If access-token is defined, extract user from it. - if dc.serverDetails.AccessToken != "" { + if details.AccessToken != "" { log.Debug("Using access-token details for nuget authentication.") if user == "" { - user = auth.ExtractUsernameFromAccessToken(dc.serverDetails.AccessToken) + user = auth.ExtractUsernameFromAccessToken(details.AccessToken) } - password = dc.serverDetails.AccessToken + password = details.AccessToken } return } diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index 80dd7cf3e..2875b1290 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -154,11 +154,7 @@ func (gc *GoCommand) run() error { if err != nil { return err } - serverDetails, err := resolverDetails.CreateArtAuthConfig() - if err != nil { - return err - } - repoUrl, err := getArtifactoryApiUrl(gc.resolverParams.TargetRepo(), serverDetails) + repoUrl, err := GetGoRepoUrl(resolverDetails, gc.resolverParams.TargetRepo()) if err != nil { return err } @@ -190,6 +186,10 @@ func (gc *GoCommand) run() error { err = e } }() + serverDetails, err := resolverDetails.CreateArtAuthConfig() + if err != nil { + return err + } err = copyGoPackageFiles(tempDirPath, gc.goArg[1], gc.resolverParams.TargetRepo(), serverDetails) if err != nil { return err @@ -208,6 +208,15 @@ func (gc *GoCommand) run() error { return err } +func GetGoRepoUrl(serverDetails *config.ServerDetails, repo string) (string, error) { + authServerDetails, err := serverDetails.CreateArtAuthConfig() + if err != nil { + return "", err + } + return getArtifactoryApiUrl(repo, authServerDetails) + +} + // Gets the URL of the specified repository Go API in Artifactory. // The URL contains credentials (username and access token or password). func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails) (string, error) { diff --git a/artifactory/commands/golang/testdata/.gitignore b/artifactory/commands/golang/testdata/.gitignore index ba7ea82da..2dd308744 100644 --- a/artifactory/commands/golang/testdata/.gitignore +++ b/artifactory/commands/golang/testdata/.gitignore @@ -8,5 +8,5 @@ # Test binary, build with `go test -c` *.test -# Output of the go coverage tool, specifically when used with LiteIDE +# Output of the go coverage Tool, specifically when used with LiteIDE *.out \ No newline at end of file diff --git a/artifactory/commands/gradle/gradle.go b/artifactory/commands/gradle/gradle.go index c2a16e94a..ba8eb7a6e 100644 --- a/artifactory/commands/gradle/gradle.go +++ b/artifactory/commands/gradle/gradle.go @@ -56,7 +56,7 @@ func (gc *GradleCommand) SetServerDetails(serverDetails *config.ServerDetails) * func (gc *GradleCommand) init() (vConfig *viper.Viper, err error) { // Read config - vConfig, err = utils.ReadGradleConfig(gc.configPath, false) + vConfig, err = utils.ReadGradleConfig(gc.configPath, nil) if err != nil { return } @@ -90,7 +90,7 @@ func (gc *GradleCommand) Run() error { if err != nil { return err } - err = gradleutils.RunGradle(vConfig, gc.tasks, gc.buildArtifactsDetailsFile, gc.configuration, gc.threads, false, gc.IsXrayScan()) + err = gradleutils.RunGradle(vConfig, gc.tasks, gc.buildArtifactsDetailsFile, gc.configuration, gc.threads, gc.IsXrayScan()) if err != nil { return err } diff --git a/artifactory/commands/mvn/mvn.go b/artifactory/commands/mvn/mvn.go index 1ad571d31..4a8bb6cfe 100644 --- a/artifactory/commands/mvn/mvn.go +++ b/artifactory/commands/mvn/mvn.go @@ -97,7 +97,7 @@ func (mc *MvnCommand) setResult(result *commandsutils.Result) *MvnCommand { func (mc *MvnCommand) init() (vConfig *viper.Viper, err error) { // Read config - vConfig, err = utils.ReadMavenConfig(mc.configPath) + vConfig, err = utils.ReadMavenConfig(mc.configPath, nil) if err != nil { return } diff --git a/artifactory/commands/npm/common.go b/artifactory/commands/npm/common.go index 7cd2d1e9a..cc0d105c9 100644 --- a/artifactory/commands/npm/common.go +++ b/artifactory/commands/npm/common.go @@ -37,7 +37,7 @@ type CommonArgs struct { NpmCommand } -func (com *CommonArgs) preparePrerequisites(repo string, overrideNpmrc bool) error { +func (com *CommonArgs) PreparePrerequisites(repo string, overrideNpmrc bool) error { log.Debug("Preparing prerequisites...") var err error com.npmVersion, com.executablePath, err = biutils.GetNpmVersionAndExecPath(log.Logger) @@ -73,6 +73,11 @@ func (com *CommonArgs) preparePrerequisites(repo string, overrideNpmrc bool) err return com.setRestoreNpmrcFunc() } +func (com *CommonArgs) SetCmdName(cmdName string) *CommonArgs { + com.cmdName = cmdName + return com +} + func (com *CommonArgs) setJsonOutput() error { jsonOutput, err := npm.ConfigGet(com.npmArgs, "json", com.executablePath) if err != nil { @@ -98,7 +103,7 @@ func (com *CommonArgs) setArtifactoryAuth() error { // In order to make sure the npm resolves artifacts from Artifactory we create a .npmrc file in the project dir. // If such a file exists we back it up as npmrcBackupFileName. -func (com *CommonArgs) createTempNpmrc() error { +func (com *CommonArgs) CreateTempNpmrc() error { log.Debug("Creating project .npmrc file.") data, err := npm.GetConfigList(com.npmArgs, com.executablePath) if err != nil { @@ -168,3 +173,7 @@ func (com *CommonArgs) setRestoreNpmrcFunc() error { } return err } + +func (com *CommonArgs) GetRestoreNpmrcFunc() func() error { + return com.restoreNpmrcFunc +} diff --git a/artifactory/commands/npm/generic.go b/artifactory/commands/npm/generic.go index 9b9748f64..a98cedcf0 100644 --- a/artifactory/commands/npm/generic.go +++ b/artifactory/commands/npm/generic.go @@ -35,7 +35,7 @@ func (gca *GenericCommandArgs) ServerDetails() (*config.ServerDetails, error) { } func (gc *GenericCommand) Run() (err error) { - if err = gc.preparePrerequisites("", false); err != nil { + if err = gc.PreparePrerequisites("", false); err != nil { return } return gc.runNpmGenericCommand() diff --git a/artifactory/commands/npm/installorci.go b/artifactory/commands/npm/installorci.go index ff3685a96..66920a886 100644 --- a/artifactory/commands/npm/installorci.go +++ b/artifactory/commands/npm/installorci.go @@ -82,21 +82,20 @@ func (nic *NpmInstallOrCiCommand) ServerDetails() (*config.ServerDetails, error) } func (nic *NpmInstallOrCiCommand) Run() (err error) { - if err = nic.preparePrerequisites(nic.repo, true); err != nil { + if err = nic.PreparePrerequisites(nic.repo, true); err != nil { return } - - if err = nic.prepareBuildInfoModule(); err != nil { - return - } - defer func() { e := nic.restoreNpmrcFunc() if err == nil { err = e } }() - if err = nic.createTempNpmrc(); err != nil { + if err = nic.CreateTempNpmrc(); err != nil { + return + } + + if err = nic.prepareBuildInfoModule(); err != nil { return } diff --git a/artifactory/commands/python/poetry.go b/artifactory/commands/python/poetry.go index eb9f4cd25..ad3ae9870 100644 --- a/artifactory/commands/python/poetry.go +++ b/artifactory/commands/python/poetry.go @@ -11,12 +11,10 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python/dependencies" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/spf13/viper" "io" - "net/url" "os" "os/exec" "path/filepath" @@ -138,37 +136,29 @@ func (pc *PoetryCommand) SetCommandName(commandName string) *PoetryCommand { } func (pc *PoetryCommand) SetPypiRepoUrlWithCredentials() error { - rtUrl, err := url.Parse(pc.serverDetails.GetArtifactoryUrl()) + rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(pc.serverDetails, pc.repository) if err != nil { - return errorutils.CheckError(err) - } - - username := pc.serverDetails.GetUser() - password := pc.serverDetails.GetPassword() - - // Get credentials from access-token if exists. - if pc.serverDetails.GetAccessToken() != "" { - if username == "" { - username = auth.ExtractUsernameFromAccessToken(pc.serverDetails.GetAccessToken()) - } - password = pc.serverDetails.GetAccessToken() + return err } - rtUrl.Path += "api/pypi/" + pc.repository + "/simple" if password != "" { - return pc.configPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password) + return ConfigPoetryRepo( + rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, + username, + password, + pc.poetryConfigRepoName) } return nil } -func (pc *PoetryCommand) configPoetryRepo(url, username, password string) error { +func ConfigPoetryRepo(url, username, password, configRepoName string) error { // Add the poetry repository config - err := runPoetryConfigCommand([]string{poetryConfigRepoPrefix + pc.poetryConfigRepoName, url}, false) + err := runPoetryConfigCommand([]string{poetryConfigRepoPrefix + configRepoName, url}, false) if err != nil { return err } // Set the poetry repository credentials - err = runPoetryConfigCommand([]string{poetryConfigAuthPrefix + pc.poetryConfigRepoName, username, password}, true) + err = runPoetryConfigCommand([]string{poetryConfigAuthPrefix + configRepoName, username, password}, true) if err != nil { return err } @@ -178,7 +168,7 @@ func (pc *PoetryCommand) configPoetryRepo(url, username, password string) error if err != nil { return errorutils.CheckError(err) } - return addRepoToPyprojectFile(filepath.Join(currentDir, pyproject), pc.poetryConfigRepoName, url) + return addRepoToPyprojectFile(filepath.Join(currentDir, pyproject), configRepoName, url) } func addRepoToPyprojectFile(filepath, poetryRepoName, repoUrl string) error { diff --git a/artifactory/commands/python/python.go b/artifactory/commands/python/python.go index 9f0a981c5..17c0e8911 100644 --- a/artifactory/commands/python/python.go +++ b/artifactory/commands/python/python.go @@ -18,6 +18,11 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) +const ( + PipenvRemoteRegistryFlag = "--pypi-mirror" + PipRemoteRegistryFlag = "-i" +) + type PythonCommand struct { serverDetails *config.ServerDetails pythonTool pythonutils.PythonTool @@ -103,32 +108,11 @@ func (pc *PythonCommand) SetCommandName(commandName string) *PythonCommand { } func (pc *PythonCommand) SetPypiRepoUrlWithCredentials() error { - rtUrl, err := url.Parse(pc.serverDetails.GetArtifactoryUrl()) + rtUrl, err := GetPypiRepoUrl(pc.serverDetails, pc.repository) if err != nil { - return errorutils.CheckError(err) - } - - username := pc.serverDetails.GetUser() - password := pc.serverDetails.GetPassword() - - // Get credentials from access-token if exists. - if pc.serverDetails.GetAccessToken() != "" { - if username == "" { - username = auth.ExtractUsernameFromAccessToken(pc.serverDetails.GetAccessToken()) - } - password = pc.serverDetails.GetAccessToken() - } - if password != "" { - rtUrl.User = url.UserPassword(username, password) - } - rtUrl.Path += "api/pypi/" + pc.repository + "/simple" - - if pc.pythonTool == pythonutils.Pip { - pc.args = append(pc.args, "-i") - } else if pc.pythonTool == pythonutils.Pipenv { - pc.args = append(pc.args, "--pypi-mirror") + return err } - pc.args = append(pc.args, rtUrl.String()) + pc.args = append(pc.args, GetPypiRemoteRegistryFlag(pc.pythonTool), rtUrl.String()) return nil } @@ -160,3 +144,41 @@ func (pc *PythonCommand) GetStdWriter() io.WriteCloser { func (pc *PythonCommand) GetErrWriter() io.WriteCloser { return nil } + +func GetPypiRepoUrl(serverDetails *config.ServerDetails, repository string) (*url.URL, error) { + rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(serverDetails, repository) + if err != nil { + return nil, err + } + if password != "" { + rtUrl.User = url.UserPassword(username, password) + } + return rtUrl, err +} + +func GetPypiRepoUrlWithCredentials(serverDetails *config.ServerDetails, repository string) (*url.URL, string, string, error) { + rtUrl, err := url.Parse(serverDetails.GetArtifactoryUrl()) + if err != nil { + return nil, "", "", errorutils.CheckError(err) + } + + username := serverDetails.GetUser() + password := serverDetails.GetPassword() + + // Get credentials from access-token if exists. + if serverDetails.GetAccessToken() != "" { + if username == "" { + username = auth.ExtractUsernameFromAccessToken(serverDetails.GetAccessToken()) + } + password = serverDetails.GetAccessToken() + } + rtUrl.Path += "api/pypi/" + repository + "/simple" + return rtUrl, username, password, err +} + +func GetPypiRemoteRegistryFlag(tool pythonutils.PythonTool) string { + if tool == pythonutils.Pip { + return PipRemoteRegistryFlag + } + return PipenvRemoteRegistryFlag +} diff --git a/artifactory/commands/yarn/yarn.go b/artifactory/commands/yarn/yarn.go index c171d046f..502a8306c 100644 --- a/artifactory/commands/yarn/yarn.go +++ b/artifactory/commands/yarn/yarn.go @@ -21,9 +21,14 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -const yarnrcFileName = ".yarnrc.yml" -const yarnrcBackupFileName = "jfrog.yarnrc.backup" -const npmScopesConfigName = "npmScopes" +const ( + YarnrcFileName = ".yarnrc.yml" + YarnrcBackupFileName = "jfrog.yarnrc.backup" + NpmScopesConfigName = "npmScopes" + yarnNpmRegistryServerEnv = "" + yarnNpmAuthIndent = "YARN_NPM_AUTH_IDENT" + yarnNpmAlwaysAuth = "YARN_NPM_ALWAYS_AUTH" +) type YarnCommand struct { executablePath string @@ -35,11 +40,9 @@ type YarnCommand struct { configFilePath string yarnArgs []string threads int - restoreYarnrcFunc func() error serverDetails *config.ServerDetails authArtDetails auth.ServiceDetails buildConfiguration *utils.BuildConfiguration - envVarsBackup map[string]*string buildInfoModule *build.YarnModule } @@ -92,18 +95,19 @@ func (yc *YarnCommand) Run() error { }() } - yc.restoreYarnrcFunc, err = commandUtils.BackupFile(filepath.Join(yc.workingDirectory, yarnrcFileName), filepath.Join(yc.workingDirectory, yarnrcBackupFileName)) + restoreYarnrcFunc, err := commandUtils.BackupFile(filepath.Join(yc.workingDirectory, YarnrcFileName), filepath.Join(yc.workingDirectory, YarnrcBackupFileName)) if err != nil { - return yc.restoreConfigurationsAndError(err) + return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } - if err = yc.modifyYarnConfigurations(); err != nil { - return yc.restoreConfigurationsAndError(err) + backupEnvMap, err := ModifyYarnConfigurations(yc.executablePath, yc.registry, yc.npmAuthIdent) + if err != nil { + return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } yc.buildInfoModule.SetArgs(filteredYarnArgs) if err = yc.buildInfoModule.Build(); err != nil { - return yc.restoreConfigurationsAndError(err) + return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } if yc.collectBuildInfo { @@ -111,7 +115,7 @@ func (yc *YarnCommand) Run() error { commandUtils.PrintMissingDependencies(missingDependencies) } - if err = yc.restoreConfigurationsFromBackup(); err != nil { + if err = RestoreConfigurationsFromBackup(backupEnvMap, restoreYarnrcFunc); err != nil { return err } @@ -161,6 +165,13 @@ func (yc *YarnCommand) readConfigFile() error { return err } +func RestoreConfigurationsAndError(envVarsBackup map[string]*string, restoreNpmrcFunc func() error, err error) error { + if restoreErr := RestoreConfigurationsFromBackup(envVarsBackup, restoreNpmrcFunc); restoreErr != nil { + return fmt.Errorf("two errors occurred:\n%s\n%s", restoreErr.Error(), err.Error()) + } + return err +} + func (yc *YarnCommand) preparePrerequisites() error { log.Debug("Preparing prerequisites.") var err error @@ -201,20 +212,31 @@ func (yc *YarnCommand) preparePrerequisites() error { yc.buildInfoModule.SetName(yc.buildConfiguration.GetModule()) } - if err = yc.setArtifactoryAuth(); err != nil { - return err + yc.registry, yc.npmAuthIdent, err = GetYarnAuthDetails(yc.serverDetails, yc.registry) + return err +} + +func (yc *YarnCommand) prepareBuildInfo() (missingDepsChan chan string, err error) { + log.Info("Preparing for dependencies information collection... For the first run of the build, the dependencies collection may take a few minutes. Subsequent runs should be faster.") + servicesManager, err := utils.CreateServiceManager(yc.serverDetails, -1, 0, false) + if err != nil { + return } - var npmAuthOutput string - npmAuthOutput, yc.registry, err = commandUtils.GetArtifactoryNpmRepoDetails(yc.repo, &yc.authArtDetails) + // Collect checksums from last build to decrease requests to Artifactory + buildName, err := yc.buildConfiguration.GetBuildName() if err != nil { - return err + return } - yc.npmAuthIdent, err = extractAuthIdentFromNpmAuth(npmAuthOutput) + previousBuildDependencies, err := commandUtils.GetDependenciesFromLatestBuild(servicesManager, buildName) if err != nil { - return err + return } - return nil + missingDepsChan = make(chan string) + collectChecksumsFunc := commandUtils.CreateCollectChecksumsFunc(previousBuildDependencies, servicesManager, missingDepsChan) + yc.buildInfoModule.SetTraverseDependenciesFunc(collectChecksumsFunc) + yc.buildInfoModule.SetThreads(yc.threads) + return } func (yc *YarnCommand) setYarnExecutable() error { @@ -228,34 +250,43 @@ func (yc *YarnCommand) setYarnExecutable() error { return nil } -func (yc *YarnCommand) setArtifactoryAuth() error { - authArtDetails, err := yc.serverDetails.CreateArtAuthConfig() +func GetYarnAuthDetails(server *config.ServerDetails, repo string) (string, string, error) { + authRtDetails, err := setArtifactoryAuth(server) if err != nil { - return err + return "", "", err } - if authArtDetails.GetSshAuthHeaders() != nil { - return errorutils.CheckErrorf("SSH authentication is not supported in this command") + var npmAuthOutput string + npmAuthOutput, registry, err := commandUtils.GetArtifactoryNpmRepoDetails(repo, &authRtDetails) + if err != nil { + return "", "", err } - yc.authArtDetails = authArtDetails - return nil + npmAuthIdent, err := extractAuthIdentFromNpmAuth(npmAuthOutput) + if err != nil { + return "", "", err + } + return registry, npmAuthIdent, nil } -func (yc *YarnCommand) restoreConfigurationsFromBackup() error { - if err := yc.restoreEnvironmentVariables(); err != nil { - return err +func setArtifactoryAuth(server *config.ServerDetails) (auth.ServiceDetails, error) { + authArtDetails, err := server.CreateArtAuthConfig() + if err != nil { + return nil, err + } + if authArtDetails.GetSshAuthHeaders() != nil { + return nil, errorutils.CheckErrorf("SSH authentication is not supported in this command") } - return yc.restoreYarnrcFunc() + return authArtDetails, nil } -func (yc *YarnCommand) restoreConfigurationsAndError(err error) error { - if restoreErr := yc.restoreConfigurationsFromBackup(); restoreErr != nil { - return fmt.Errorf("two errors occurred:\n%s\n%s", restoreErr.Error(), err.Error()) +func RestoreConfigurationsFromBackup(envVarsBackup map[string]*string, restoreYarnrcFunc func() error) error { + if err := restoreEnvironmentVariables(envVarsBackup); err != nil { + return err } - return err + return restoreYarnrcFunc() } -func (yc *YarnCommand) restoreEnvironmentVariables() error { - for key, value := range yc.envVarsBackup { +func restoreEnvironmentVariables(envVarsBackup map[string]*string) error { + for key, value := range envVarsBackup { if value == nil { if err := os.Unsetenv(key); err != nil { return err @@ -270,23 +301,26 @@ func (yc *YarnCommand) restoreEnvironmentVariables() error { return nil } -func (yc *YarnCommand) modifyYarnConfigurations() error { - yc.envVarsBackup = make(map[string]*string) - - if err := yc.backupAndSetEnvironmentVariable("YARN_NPM_REGISTRY_SERVER", yc.registry); err != nil { - return err - } - - if err := yc.backupAndSetEnvironmentVariable("YARN_NPM_AUTH_IDENT", yc.npmAuthIdent); err != nil { - return err +func ModifyYarnConfigurations(execPath, registry, npmAuthIdent string) (map[string]*string, error) { + envVarsUpdated := map[string]string{ + yarnNpmRegistryServerEnv: registry, + yarnNpmAuthIndent: npmAuthIdent, + yarnNpmAlwaysAuth: "true", } - - if err := yc.backupAndSetEnvironmentVariable("YARN_NPM_ALWAYS_AUTH", "true"); err != nil { - return err + envVarsBackup := make(map[string]*string) + for key, value := range envVarsUpdated { + oldVal, err := backupAndSetEnvironmentVariable(key, value) + if err != nil { + return nil, err + } + envVarsBackup[key] = &oldVal } - // Update scoped registries (these cannot be set in environment variables) - npmScopesStr, err := yarn.ConfigGet(npmScopesConfigName, yc.executablePath, true) + return envVarsBackup, errorutils.CheckError(UpdateScopeRegistries(execPath, registry, npmAuthIdent)) +} + +func UpdateScopeRegistries(execPath, registry, npmAuthIdent string) error { + npmScopesStr, err := yarn.ConfigGet(NpmScopesConfigName, execPath, true) if err != nil { return err } @@ -295,7 +329,7 @@ func (yc *YarnCommand) modifyYarnConfigurations() error { if err != nil { return errorutils.CheckError(err) } - artifactoryScope := yarnNpmScope{NpmAlwaysAuth: true, NpmAuthIdent: yc.npmAuthIdent, NpmRegistryServer: yc.registry} + artifactoryScope := yarnNpmScope{NpmAlwaysAuth: true, NpmAuthIdent: npmAuthIdent, NpmRegistryServer: registry} for scopeName := range npmScopesMap { npmScopesMap[scopeName] = artifactoryScope } @@ -303,8 +337,7 @@ func (yc *YarnCommand) modifyYarnConfigurations() error { if err != nil { return errorutils.CheckError(err) } - err = yarn.ConfigSet(npmScopesConfigName, string(updatedNpmScopesStr), yc.executablePath, true) - return errorutils.CheckError(err) + return yarn.ConfigSet(NpmScopesConfigName, string(updatedNpmScopesStr), execPath, true) } type yarnNpmScope struct { @@ -313,38 +346,9 @@ type yarnNpmScope struct { NpmRegistryServer string `json:"npmRegistryServer,omitempty"` } -func (yc *YarnCommand) backupAndSetEnvironmentVariable(key, value string) error { - oldVal, exist := os.LookupEnv(key) - if exist { - yc.envVarsBackup[key] = &oldVal - } else { - yc.envVarsBackup[key] = nil - } - - return errorutils.CheckError(os.Setenv(key, value)) -} - -func (yc *YarnCommand) prepareBuildInfo() (missingDepsChan chan string, err error) { - log.Info("Preparing for dependencies information collection... For the first run of the build, the dependencies collection may take a few minutes. Subsequent runs should be faster.") - servicesManager, err := utils.CreateServiceManager(yc.serverDetails, -1, 0, false) - if err != nil { - return - } - - // Collect checksums from last build to decrease requests to Artifactory - buildName, err := yc.buildConfiguration.GetBuildName() - if err != nil { - return - } - previousBuildDependencies, err := commandUtils.GetDependenciesFromLatestBuild(servicesManager, buildName) - if err != nil { - return - } - missingDepsChan = make(chan string) - collectChecksumsFunc := commandUtils.CreateCollectChecksumsFunc(previousBuildDependencies, servicesManager, missingDepsChan) - yc.buildInfoModule.SetTraverseDependenciesFunc(collectChecksumsFunc) - yc.buildInfoModule.SetThreads(yc.threads) - return +func backupAndSetEnvironmentVariable(key, value string) (string, error) { + oldVal, _ := os.LookupEnv(key) + return oldVal, errorutils.CheckError(os.Setenv(key, value)) } // npmAuth we get back from Artifactory includes several fields, but we need only the field '_auth' diff --git a/artifactory/utils/buildinfoproperties.go b/artifactory/utils/buildinfoproperties.go index 1e24778a6..877588f54 100644 --- a/artifactory/utils/buildinfoproperties.go +++ b/artifactory/utils/buildinfoproperties.go @@ -155,36 +155,31 @@ func ReadConfigFile(configPath string, configType ConfigType) (config *viper.Vip return config, errorutils.CheckError(err) } -func ReadGradleConfig(path string, useWrapperIfMissingConfig bool) (config *viper.Viper, err error) { +func ReadGradleConfig(path string, gradleConfigParams map[string]any) (config *viper.Viper, err error) { if path == "" { - config = createDefaultGradleConfig(useWrapperIfMissingConfig) + config = createDefaultConfigWithParams(YAML, Gradle.String(), gradleConfigParams) } else { config, err = ReadConfigFile(path, YAML) } return } -func ReadMavenConfig(path string) (config *viper.Viper, err error) { +func ReadMavenConfig(path string, mvnProps map[string]any) (config *viper.Viper, err error) { if path == "" { - config = createDefaultMavenConfig() + config = createDefaultConfigWithParams(YAML, Maven.String(), mvnProps) } else { config, err = ReadConfigFile(path, YAML) } return } -func createDefaultMavenConfig() *viper.Viper { +func createDefaultConfigWithParams(configType ConfigType, technology string, params map[string]any) *viper.Viper { vConfig := viper.New() - vConfig.SetConfigType(string(YAML)) - vConfig.Set("type", Maven.String()) - return vConfig -} - -func createDefaultGradleConfig(useWrapperIfMissingConfig bool) *viper.Viper { - vConfig := viper.New() - vConfig.SetConfigType(string(YAML)) - vConfig.Set("type", Gradle.String()) - vConfig.Set("usewrapper", useWrapperIfMissingConfig) + vConfig.SetConfigType(string(configType)) + vConfig.Set("type", technology) + for key, value := range params { + vConfig.Set(key, value) + } return vConfig } diff --git a/utils/mvn/utils.go b/utils/mvn/utils.go index 79c0a377d..95a4a3921 100644 --- a/utils/mvn/utils.go +++ b/utils/mvn/utils.go @@ -14,7 +14,8 @@ import ( "github.com/spf13/viper" ) -func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration, goals []string, threads int, insecureTls, disableDeploy bool) error { +func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration, + goals []string, threads int, insecureTls, disableDeploy bool) error { buildInfoService := utils.CreateBuildInfoService() buildName, err := buildConf.GetBuildName() if err != nil { diff --git a/xray/audit/go/golang.go b/xray/audit/go/golang.go index 67949e0e2..f4fb7dcb9 100644 --- a/xray/audit/go/golang.go +++ b/xray/audit/go/golang.go @@ -2,6 +2,9 @@ package _go import ( "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/golang" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "os" "strings" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -14,11 +17,16 @@ const ( goSourceCodePrefix = "github.com/golang/go:v" ) -func BuildDependencyTree() (dependencyTree []*services.GraphNode, err error) { +func BuildDependencyTree(server *config.ServerDetails, remoteGoRepo string) (dependencyTree []*services.GraphNode, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return } + if remoteGoRepo != "" { + if err = setGoProxy(server, remoteGoRepo); err != nil { + return + } + } // Calculate go dependencies graph dependenciesGraph, err := goutils.GetDependenciesGraph(currentDir) if err != nil { @@ -51,6 +59,14 @@ func BuildDependencyTree() (dependencyTree []*services.GraphNode, err error) { return } +func setGoProxy(server *config.ServerDetails, remoteGoRepo string) error { + repoUrl, err := golang.GetGoRepoUrl(server, remoteGoRepo) + if err != nil { + return err + } + return os.Setenv("GOPROXY", repoUrl) +} + func populateGoDependencyTree(currNode *services.GraphNode, dependenciesGraph map[string][]string, dependenciesList map[string]bool) { if currNode.NodeHasLoop() { return diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index b4721b79a..079838e80 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -12,11 +12,11 @@ import ( const gradlew = "gradlew" -func BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile bool) (dependencyTree []*services.GraphNode, err error) { +func BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (dependencyTree []*services.GraphNode, err error) { buildConfiguration, cleanBuild := createBuildConfiguration("audit-gradle") defer cleanBuild(err) - err = runGradle(buildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile) + err = runGradle(buildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile, gradleConfigParams) if err != nil { return } @@ -25,7 +25,7 @@ func BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile boo return } -func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile bool) (err error) { +func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (err error) { tasks := "clean compileJava " if !excludeTestDeps { tasks += "compileTestJava " @@ -49,9 +49,10 @@ func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, us if err != nil { return } + gradleConfigParams["usewrapper"] = useWrapper } // Read config - vConfig, err := utils.ReadGradleConfig(configFilePath, useWrapper) + vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams) if err != nil { return err } diff --git a/xray/audit/java/javautils.go b/xray/audit/java/javautils.go index 509b5d68f..b657ef8fa 100644 --- a/xray/audit/java/javautils.go +++ b/xray/audit/java/javautils.go @@ -1,6 +1,7 @@ package java import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "strconv" "time" @@ -15,6 +16,15 @@ const ( GavPackageTypeIdentifier = "gav://" ) +type DependencyTreeParams struct { + Tool coreutils.Technology + InsecureTls bool + IgnoreConfigFile bool + ExcludeTestDeps bool + UseWrapper bool + JavaProps map[string]any +} + func createBuildConfiguration(buildName string) (*artifactoryUtils.BuildConfiguration, func(err error)) { buildConfiguration := artifactoryUtils.NewBuildConfiguration(buildName, strconv.FormatInt(time.Now().Unix(), 10), "", "") return buildConfiguration, func(err error) { @@ -124,6 +134,13 @@ func hasLoop(idsAdded []string, idToAdd string) bool { return false } +func BuildDependencyTree(params *DependencyTreeParams) (modules []*services.GraphNode, err error) { + if params.Tool == coreutils.Maven { + return BuildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.JavaProps) + } + return BuildGradleDependencyTree(params.ExcludeTestDeps, params.UseWrapper, params.IgnoreConfigFile, params.JavaProps) +} + type dependencyMultimap struct { multimap map[string]map[string]*buildinfo.Dependency } diff --git a/xray/audit/java/mvn.go b/xray/audit/java/mvn.go index 2c6706ea9..c544feeb6 100644 --- a/xray/audit/java/mvn.go +++ b/xray/audit/java/mvn.go @@ -9,11 +9,11 @@ import ( "github.com/jfrog/jfrog-client-go/xray/services" ) -func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool) (modules []*services.GraphNode, err error) { +func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool, mvnProps map[string]any) (modules []*services.GraphNode, err error) { buildConfiguration, cleanBuild := createBuildConfiguration("audit-mvn") defer cleanBuild(err) - err = runMvn(buildConfiguration, insecureTls, ignoreConfigFile) + err = runMvn(buildConfiguration, insecureTls, ignoreConfigFile, mvnProps) if err != nil { return } @@ -21,7 +21,7 @@ func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool) (modules []*serv return createGavDependencyTree(buildConfiguration) } -func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile bool) (err error) { +func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile bool, mvnProps map[string]any) (err error) { goals := []string{"-B", "compile", "test-compile"} log.Debug(fmt.Sprintf("mvn command goals: %v", goals)) configFilePath := "" @@ -36,7 +36,7 @@ func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreCon } } // Read config - vConfig, err := utils.ReadMavenConfig(configFilePath) + vConfig, err := utils.ReadMavenConfig(configFilePath, mvnProps) if err != nil { return err } diff --git a/xray/audit/java/mvn_test.go b/xray/audit/java/mvn_test.go index 15e8a47cc..ced00dd70 100644 --- a/xray/audit/java/mvn_test.go +++ b/xray/audit/java/mvn_test.go @@ -14,7 +14,7 @@ func TestMavenTreesMultiModule(t *testing.T) { defer cleanUp() // Run getModulesDependencyTrees - modulesDependencyTrees, err := BuildMvnDependencyTree(false, true) + modulesDependencyTrees, err := BuildMvnDependencyTree(false, true, nil) if assert.NoError(t, err) && assert.NotEmpty(t, modulesDependencyTrees) { // Check root module multi := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test:multi:3.7-SNAPSHOT") diff --git a/xray/audit/python/python.go b/xray/audit/python/python.go index 604ff7e5b..4dc83ab10 100644 --- a/xray/audit/python/python.go +++ b/xray/audit/python/python.go @@ -3,6 +3,7 @@ package python import ( "fmt" "github.com/jfrog/build-info-go/utils/pythonutils" + rtpython "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/xray/audit" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -20,8 +21,15 @@ const ( pythonPackageTypeIdentifier = "pypi://" ) -func BuildDependencyTree(pythonTool pythonutils.PythonTool, requirementsFile string) (dependencyTree []*services.GraphNode, err error) { - dependenciesGraph, directDependenciesList, err := getDependencies(pythonTool, requirementsFile) +type AuditPython struct { + Server *config.ServerDetails + Tool pythonutils.PythonTool + RemotePypiRepo string + PipRequirementsFile string +} + +func BuildDependencyTree(auditPython *AuditPython) (dependencyTree []*services.GraphNode, err error) { + dependenciesGraph, directDependenciesList, err := getDependencies(auditPython) if err != nil { return } @@ -41,7 +49,7 @@ func BuildDependencyTree(pythonTool pythonutils.PythonTool, requirementsFile str return []*services.GraphNode{root}, nil } -func getDependencies(pythonTool pythonutils.PythonTool, requirementsFile string) (dependenciesGraph map[string][]string, directDependencies []string, err error) { +func getDependencies(auditPython *AuditPython) (dependenciesGraph map[string][]string, directDependencies []string, err error) { wd, err := os.Getwd() if errorutils.CheckError(err) != nil { return @@ -75,7 +83,7 @@ func getDependencies(pythonTool pythonutils.PythonTool, requirementsFile string) return } - restoreEnv, err := runPythonInstall(pythonTool, requirementsFile) + restoreEnv, err := runPythonInstall(auditPython) defer func() { e := restoreEnv() if err == nil { @@ -90,54 +98,82 @@ func getDependencies(pythonTool pythonutils.PythonTool, requirementsFile string) if err != nil { return } - dependenciesGraph, directDependencies, err = pythonutils.GetPythonDependencies(pythonTool, tempDirPath, localDependenciesPath) + dependenciesGraph, directDependencies, err = pythonutils.GetPythonDependencies(auditPython.Tool, tempDirPath, localDependenciesPath) if err != nil { if _, innerErr := audit.GetExecutableVersion("python"); innerErr != nil { log.Error(innerErr) } - if _, innerErr := audit.GetExecutableVersion(string(pythonTool)); innerErr != nil { + if _, innerErr := audit.GetExecutableVersion(string(auditPython.Tool)); innerErr != nil { log.Error(innerErr) } } return } -func runPythonInstall(pythonTool pythonutils.PythonTool, requirementsFile string) (restoreEnv func() error, err error) { +func runPythonInstall(auditPython *AuditPython) (restoreEnv func() error, err error) { + switch auditPython.Tool { + case pythonutils.Pip: + return installPipDeps(auditPython) + case pythonutils.Pipenv: + return installPipenvDeps(auditPython) + case pythonutils.Poetry: + return installPoetryDeps(auditPython) + } + return +} + +func installPoetryDeps(auditPython *AuditPython) (restoreEnv func() error, err error) { restoreEnv = func() error { return nil } - switch pythonTool { - case pythonutils.Pip: - restoreEnv, err = SetPipVirtualEnvPath() - if err != nil { - return - } - err = runPipInstall(requirementsFile) - if err != nil && requirementsFile == "" { - log.Debug(err.Error() + "\ntrying to install using a requirements file.") - reqErr := runPipInstall("requirements.txt") - if reqErr != nil { - // Return Pip install error and log the requirements fallback error. - log.Debug(reqErr.Error()) - } else { - err = nil + if auditPython.RemotePypiRepo != "" { + rtUrl, username, password, err := rtpython.GetPypiRepoUrlWithCredentials(auditPython.Server, auditPython.RemotePypiRepo) + if password != "" { + err = rtpython.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, auditPython.RemotePypiRepo) + if err != nil { + return restoreEnv, err } } - case pythonutils.Pipenv: - // Set virtualenv path to venv dir - err = os.Setenv("WORKON_HOME", ".jfrog") - if err != nil { - return - } - restoreEnv = func() error { - return os.Unsetenv("WORKON_HOME") - } - // Run 'pipenv install -d' - err = executeCommand("pipenv", "install", "-d") + } + // Run 'poetry install' + return restoreEnv, executeCommand("poetry", "install") +} - case pythonutils.Poetry: - // Run 'poetry install' - err = executeCommand("poetry", "install") +func installPipenvDeps(auditPython *AuditPython) (restoreEnv func() error, err error) { + // Set virtualenv path to venv dir + err = os.Setenv("WORKON_HOME", ".jfrog") + if err != nil { + return + } + restoreEnv = func() error { + return os.Unsetenv("WORKON_HOME") + } + if auditPython.RemotePypiRepo != "" { + return restoreEnv, runPipenvInstallFromRemoteRegistry(auditPython.Server, auditPython.RemotePypiRepo) + } + // Run 'pipenv install -d' + return restoreEnv, executeCommand("pipenv", "install", "-d") +} + +func installPipDeps(auditPython *AuditPython) (restoreEnv func() error, err error) { + restoreEnv, err = SetPipVirtualEnvPath() + if err != nil { + return + } + if auditPython.RemotePypiRepo != "" { + return restoreEnv, runPipInstallFromRemoteRegistry(auditPython.Server, auditPython.RemotePypiRepo, auditPython.PipRequirementsFile) + } + pipInstallArgs := getPipInstallArgs(auditPython.PipRequirementsFile) + err = runPipInstall(pipInstallArgs...) + if err != nil && auditPython.PipRequirementsFile == "" { + log.Debug(err.Error() + "\ntrying to install using a requirements file.") + reqErr := runPipInstall("requirements.txt") + if reqErr != nil { + // Return Pip install error and log the requirements fallback error. + log.Debug(reqErr.Error()) + } else { + err = nil + } } return } @@ -155,19 +191,46 @@ func executeCommand(executable string, args ...string) error { return nil } -func runPipInstall(requirementsFile string) error { +func runPipInstall(args ...string) error { // Try getting 'pip3' executable, if not found use 'pip' + pipExec := getPipExec() + return executeCommand(pipExec, args...) +} + +func getPipInstallArgs(requirementsFile string) []string { + if requirementsFile == "" { + // Run 'pip install .' + return []string{"install", "."} + } + // Run pip 'install -r requirements ' + return []string{"install", "-r", requirementsFile} +} + +func getPipExec() string { pipExec, _ := exec.LookPath("pip3") if pipExec == "" { pipExec = "pip" } - if requirementsFile == "" { - // Run 'pip install .' - return executeCommand(pipExec, "install", ".") - } else { - // Run pip 'install -r requirements ' - return executeCommand(pipExec, "install", "-r", requirementsFile) + return pipExec +} + +func runPipInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName, pipRequirementsFile string) (err error) { + rtUrl, err := rtpython.GetPypiRepoUrl(server, depsRepoName) + if err != nil { + return err + } + args := getPipInstallArgs(pipRequirementsFile) + args = append(args, rtpython.GetPypiRemoteRegistryFlag(pythonutils.Pip), rtUrl.String()) + return runPipInstall(args...) +} + +func runPipenvInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName string) (err error) { + rtUrl, err := rtpython.GetPypiRepoUrl(server, depsRepoName) + if err != nil { + return err } + args := []string{"install", "-d", rtpython.GetPypiRemoteRegistryFlag(pythonutils.Pipenv), rtUrl.String()} + return executeCommand("pipenv", args...) } // Execute virtualenv command: "virtualenv venvdir" / "python3 -m venv venvdir" and set path diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 9155df0f4..ff77eb96e 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -3,16 +3,17 @@ package audit import ( "errors" "fmt" - "github.com/jfrog/jfrog-cli-core/v2/xray/audit" + "github.com/jfrog/jfrog-cli-core/v2/xray/audit/java" "os" "path/filepath" "strings" + "github.com/jfrog/jfrog-cli-core/v2/xray/audit" + "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" _go "github.com/jfrog/jfrog-cli-core/v2/xray/audit/go" - "github.com/jfrog/jfrog-cli-core/v2/xray/audit/java" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/npm" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/nuget" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/python" @@ -31,16 +32,70 @@ type Params struct { excludeTestDeps bool insecureTls bool useWrapper bool + depsRepo string requirementsFile string technologies []string workingDirs []string args []string + installFunc func(tech string) error } func NewAuditParams() *Params { return &Params{} } +func (params *Params) InstallFunc() func(tech string) error { + return params.installFunc +} + +func (params *Params) XrayGraphScanParams() services.XrayGraphScanParams { + return params.xrayGraphScanParams +} + +func (params *Params) ServerDetails() *config.ServerDetails { + return params.serverDetails +} + +func (params *Params) Progress() ioUtils.ProgressMgr { + return params.progress +} + +func (params *Params) IgnoreConfigFile() bool { + return params.ignoreConfigFile +} + +func (params *Params) ExcludeTestDeps() bool { + return params.excludeTestDeps +} + +func (params *Params) InsecureTls() bool { + return params.insecureTls +} + +func (params *Params) UseWrapper() bool { + return params.useWrapper +} + +func (params *Params) DepsRepo() string { + return params.depsRepo +} + +func (params *Params) RequirementsFile() string { + return params.requirementsFile +} + +func (params *Params) Technologies() []string { + return params.technologies +} + +func (params *Params) WorkingDirs() []string { + return params.workingDirs +} + +func (params *Params) Args() []string { + return params.args +} + func (params *Params) SetXrayGraphScanParams(xrayGraphScanParams services.XrayGraphScanParams) *Params { params.xrayGraphScanParams = xrayGraphScanParams return params @@ -96,6 +151,16 @@ func (params *Params) SetTechnologies(technologies ...string) *Params { return params } +func (params *Params) SetDepsRepo(depsRepo string) *Params { + params.depsRepo = depsRepo + return params +} + +func (params *Params) SetInstallFunc(installFunc func(tech string) error) *Params { + params.installFunc = installFunc + return params +} + // GenericAudit audits all the projects found in the given workingDirs func GenericAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bool, err error) { if len(params.workingDirs) == 0 { @@ -187,18 +252,20 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc params.progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) } switch tech { - case coreutils.Maven: - dependencyTrees, e = java.BuildMvnDependencyTree(params.insecureTls, params.ignoreConfigFile) - case coreutils.Gradle: - dependencyTrees, e = java.BuildGradleDependencyTree(params.excludeTestDeps, params.useWrapper, params.ignoreConfigFile) + case coreutils.Maven, coreutils.Gradle: + dependencyTrees, e = getJavaDependencyTree(params, tech) case coreutils.Npm: dependencyTrees, e = npm.BuildDependencyTree(params.args) case coreutils.Yarn: dependencyTrees, e = yarn.BuildDependencyTree() case coreutils.Go: - dependencyTrees, e = _go.BuildDependencyTree() + dependencyTrees, e = _go.BuildDependencyTree(params.serverDetails, params.depsRepo) case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: - dependencyTrees, e = python.BuildDependencyTree(pythonutils.PythonTool(tech), params.requirementsFile) + dependencyTrees, e = python.BuildDependencyTree(&python.AuditPython{ + Server: params.serverDetails, + Tool: pythonutils.PythonTool(tech), + RemotePypiRepo: params.depsRepo, + PipRequirementsFile: params.requirementsFile}) case coreutils.Nuget: dependencyTrees, e = nuget.BuildDependencyTree() default: @@ -208,6 +275,37 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc return dependencyTrees, e } +func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*services.GraphNode, error) { + var javaProps map[string]any + if params.DepsRepo() != "" { + javaProps = createJavaProps(params.DepsRepo(), params.ServerDetails()) + } + return java.BuildDependencyTree(&java.DependencyTreeParams{ + Tool: tech, + InsecureTls: params.insecureTls, + IgnoreConfigFile: params.ignoreConfigFile, + ExcludeTestDeps: params.excludeTestDeps, + UseWrapper: params.useWrapper, + JavaProps: javaProps, + }) +} + +func createJavaProps(depsRepo string, serverDetails *config.ServerDetails) map[string]any { + authPass := serverDetails.Password + if serverDetails.AccessToken != "" { + authPass = serverDetails.AccessToken + } + return map[string]any{ + "resolver.username": serverDetails.User, + "resolver.password": authPass, + "resolver.url": serverDetails.ArtifactoryUrl, + "resolver.releaseRepo": depsRepo, + "resolver.repo": depsRepo, + "resolver.snapshotRepo": depsRepo, + "buildInfoConfig.artifactoryResolutionEnabled": true, + } +} + func detectedTechnologies() (technologies []string, err error) { wd, err := os.Getwd() if errorutils.CheckError(err) != nil { diff --git a/xray/commands/testdata/poetry-project/pyproject.toml b/xray/commands/testdata/poetry-project/pyproject.toml index 52ea7daf4..045c50e5d 100644 --- a/xray/commands/testdata/poetry-project/pyproject.toml +++ b/xray/commands/testdata/poetry-project/pyproject.toml @@ -1,14 +1,14 @@ -[tool.poetry] +[Tool.poetry] name = "my-poetry-project" version = "0.1.0" description = "" authors = ["Tal Arian "] -[tool.poetry.dependencies] +[Tool.poetry.dependencies] python = "^3.8" numpy = "^1.23.0" -[tool.poetry.dev-dependencies] +[Tool.poetry.dev-dependencies] pytest = "^5.2" [build-system] From 61420fcc240bf658aa0584ca0c1b58f49359868c Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 13 Feb 2023 15:46:25 +0200 Subject: [PATCH 03/48] refactor params Improve deps resolution and refactor audit --- artifactory/commands/dotnet/dotnetcommand.go | 30 +-- artifactory/commands/golang/go.go | 18 +- .../commands/golang/testdata/.gitignore | 2 +- artifactory/commands/gradle/gradle.go | 4 +- artifactory/commands/mvn/mvn.go | 2 +- artifactory/commands/npm/common.go | 13 +- artifactory/commands/npm/generic.go | 2 +- artifactory/commands/npm/installorci.go | 13 +- artifactory/commands/python/poetry.go | 32 ++-- artifactory/commands/python/python.go | 70 ++++--- artifactory/commands/yarn/yarn.go | 174 +++++++++--------- artifactory/utils/buildinfoproperties.go | 25 +-- utils/mvn/utils.go | 3 +- xray/audit/go/golang.go | 19 +- xray/audit/java/gradle.go | 9 +- xray/audit/java/javautils.go | 17 ++ xray/audit/java/mvn.go | 8 +- xray/audit/java/mvn_test.go | 2 +- xray/audit/python/python.go | 149 ++++++++++----- xray/commands/audit/generic/auditmanager.go | 114 +++++++++++- .../testdata/poetry-project/pyproject.toml | 6 +- 21 files changed, 468 insertions(+), 244 deletions(-) diff --git a/artifactory/commands/dotnet/dotnetcommand.go b/artifactory/commands/dotnet/dotnetcommand.go index d18ea34b4..55e5f1a3e 100644 --- a/artifactory/commands/dotnet/dotnetcommand.go +++ b/artifactory/commands/dotnet/dotnetcommand.go @@ -219,7 +219,7 @@ func (dc *DotnetCommand) prepareConfigFile() (cleanup func() error, err error) { return fileutils.RemoveTempDir(tempDirPath) } - configFile, err := dc.InitNewConfig(tempDirPath) + configFile, err := InitNewConfig(tempDirPath, dc.repoName, dc.serverDetails, dc.useNugetV2) if err == nil { dc.argAndFlags = append(dc.argAndFlags, dc.GetToolchain().GetTypeFlagPrefix()+"configfile", configFile.Name()) } @@ -242,7 +242,7 @@ func getFlagValueIfExists(cmdFlag string, argAndFlags []string) (string, error) } // The fact that we here, means that neither of the flags were provided, and we need to init our own config. -func (dc *DotnetCommand) InitNewConfig(configDirPath string) (configFile *os.File, err error) { +func InitNewConfig(configDirPath, repoName string, server *config.ServerDetails, useNugetV2 bool) (configFile *os.File, err error) { // Initializing a new NuGet config file that NuGet will use into a temp file configFile, err = os.CreateTemp(configDirPath, configFilePattern) if errorutils.CheckError(err) != nil { @@ -259,20 +259,20 @@ func (dc *DotnetCommand) InitNewConfig(configDirPath string) (configFile *os.Fil // We would prefer to write the NuGet configuration using the `nuget add source` command, // but the NuGet configuration utility doesn't currently allow setting protocolVersion. // Until that is supported, the templated method must be used. - err = dc.addSourceToNugetTemplate(configFile) + err = addSourceToNugetTemplate(configFile, server, useNugetV2, repoName) return } // Adds a source to the nuget config template -func (dc *DotnetCommand) addSourceToNugetTemplate(configFile *os.File) error { - sourceUrl, user, password, err := dc.getSourceDetails() +func addSourceToNugetTemplate(configFile *os.File, server *config.ServerDetails, useNugetV2 bool, repoName string) error { + sourceUrl, user, password, err := GetSourceDetails(server, repoName, useNugetV2) if err != nil { return err } // Specify the protocolVersion protoVer := "3" - if dc.useNugetV2 { + if useNugetV2 { protoVer = "2" } @@ -281,28 +281,28 @@ func (dc *DotnetCommand) addSourceToNugetTemplate(configFile *os.File) error { return err } -func (dc *DotnetCommand) getSourceDetails() (sourceURL, user, password string, err error) { +func GetSourceDetails(details *config.ServerDetails, repoName string, useNugetV2 bool) (sourceURL, user, password string, err error) { var u *url.URL - u, err = url.Parse(dc.serverDetails.ArtifactoryUrl) + u, err = url.Parse(details.ArtifactoryUrl) if errorutils.CheckError(err) != nil { return } nugetApi := "api/nuget/v3" - if dc.useNugetV2 { + if useNugetV2 { nugetApi = "api/nuget" } - u.Path = path.Join(u.Path, nugetApi, dc.repoName) + u.Path = path.Join(u.Path, nugetApi, repoName) sourceURL = u.String() - user = dc.serverDetails.User - password = dc.serverDetails.Password + user = details.User + password = details.Password // If access-token is defined, extract user from it. - if dc.serverDetails.AccessToken != "" { + if details.AccessToken != "" { log.Debug("Using access-token details for nuget authentication.") if user == "" { - user = auth.ExtractUsernameFromAccessToken(dc.serverDetails.AccessToken) + user = auth.ExtractUsernameFromAccessToken(details.AccessToken) } - password = dc.serverDetails.AccessToken + password = details.AccessToken } return } diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index 80dd7cf3e..4cc82cb41 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -154,11 +154,7 @@ func (gc *GoCommand) run() error { if err != nil { return err } - serverDetails, err := resolverDetails.CreateArtAuthConfig() - if err != nil { - return err - } - repoUrl, err := getArtifactoryApiUrl(gc.resolverParams.TargetRepo(), serverDetails) + repoUrl, err := GetGoRepoUrl(resolverDetails, gc.resolverParams.TargetRepo()) if err != nil { return err } @@ -190,6 +186,10 @@ func (gc *GoCommand) run() error { err = e } }() + serverDetails, err := resolverDetails.CreateArtAuthConfig() + if err != nil { + return err + } err = copyGoPackageFiles(tempDirPath, gc.goArg[1], gc.resolverParams.TargetRepo(), serverDetails) if err != nil { return err @@ -208,6 +208,14 @@ func (gc *GoCommand) run() error { return err } +func GetGoRepoUrl(serverDetails *config.ServerDetails, repo string) (string, error) { + authServerDetails, err := serverDetails.CreateArtAuthConfig() + if err != nil { + return "", err + } + return getArtifactoryApiUrl(repo, authServerDetails) +} + // Gets the URL of the specified repository Go API in Artifactory. // The URL contains credentials (username and access token or password). func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails) (string, error) { diff --git a/artifactory/commands/golang/testdata/.gitignore b/artifactory/commands/golang/testdata/.gitignore index ba7ea82da..2dd308744 100644 --- a/artifactory/commands/golang/testdata/.gitignore +++ b/artifactory/commands/golang/testdata/.gitignore @@ -8,5 +8,5 @@ # Test binary, build with `go test -c` *.test -# Output of the go coverage tool, specifically when used with LiteIDE +# Output of the go coverage Tool, specifically when used with LiteIDE *.out \ No newline at end of file diff --git a/artifactory/commands/gradle/gradle.go b/artifactory/commands/gradle/gradle.go index c2a16e94a..ba8eb7a6e 100644 --- a/artifactory/commands/gradle/gradle.go +++ b/artifactory/commands/gradle/gradle.go @@ -56,7 +56,7 @@ func (gc *GradleCommand) SetServerDetails(serverDetails *config.ServerDetails) * func (gc *GradleCommand) init() (vConfig *viper.Viper, err error) { // Read config - vConfig, err = utils.ReadGradleConfig(gc.configPath, false) + vConfig, err = utils.ReadGradleConfig(gc.configPath, nil) if err != nil { return } @@ -90,7 +90,7 @@ func (gc *GradleCommand) Run() error { if err != nil { return err } - err = gradleutils.RunGradle(vConfig, gc.tasks, gc.buildArtifactsDetailsFile, gc.configuration, gc.threads, false, gc.IsXrayScan()) + err = gradleutils.RunGradle(vConfig, gc.tasks, gc.buildArtifactsDetailsFile, gc.configuration, gc.threads, gc.IsXrayScan()) if err != nil { return err } diff --git a/artifactory/commands/mvn/mvn.go b/artifactory/commands/mvn/mvn.go index 1ad571d31..4a8bb6cfe 100644 --- a/artifactory/commands/mvn/mvn.go +++ b/artifactory/commands/mvn/mvn.go @@ -97,7 +97,7 @@ func (mc *MvnCommand) setResult(result *commandsutils.Result) *MvnCommand { func (mc *MvnCommand) init() (vConfig *viper.Viper, err error) { // Read config - vConfig, err = utils.ReadMavenConfig(mc.configPath) + vConfig, err = utils.ReadMavenConfig(mc.configPath, nil) if err != nil { return } diff --git a/artifactory/commands/npm/common.go b/artifactory/commands/npm/common.go index 7cd2d1e9a..cc0d105c9 100644 --- a/artifactory/commands/npm/common.go +++ b/artifactory/commands/npm/common.go @@ -37,7 +37,7 @@ type CommonArgs struct { NpmCommand } -func (com *CommonArgs) preparePrerequisites(repo string, overrideNpmrc bool) error { +func (com *CommonArgs) PreparePrerequisites(repo string, overrideNpmrc bool) error { log.Debug("Preparing prerequisites...") var err error com.npmVersion, com.executablePath, err = biutils.GetNpmVersionAndExecPath(log.Logger) @@ -73,6 +73,11 @@ func (com *CommonArgs) preparePrerequisites(repo string, overrideNpmrc bool) err return com.setRestoreNpmrcFunc() } +func (com *CommonArgs) SetCmdName(cmdName string) *CommonArgs { + com.cmdName = cmdName + return com +} + func (com *CommonArgs) setJsonOutput() error { jsonOutput, err := npm.ConfigGet(com.npmArgs, "json", com.executablePath) if err != nil { @@ -98,7 +103,7 @@ func (com *CommonArgs) setArtifactoryAuth() error { // In order to make sure the npm resolves artifacts from Artifactory we create a .npmrc file in the project dir. // If such a file exists we back it up as npmrcBackupFileName. -func (com *CommonArgs) createTempNpmrc() error { +func (com *CommonArgs) CreateTempNpmrc() error { log.Debug("Creating project .npmrc file.") data, err := npm.GetConfigList(com.npmArgs, com.executablePath) if err != nil { @@ -168,3 +173,7 @@ func (com *CommonArgs) setRestoreNpmrcFunc() error { } return err } + +func (com *CommonArgs) GetRestoreNpmrcFunc() func() error { + return com.restoreNpmrcFunc +} diff --git a/artifactory/commands/npm/generic.go b/artifactory/commands/npm/generic.go index 9b9748f64..a98cedcf0 100644 --- a/artifactory/commands/npm/generic.go +++ b/artifactory/commands/npm/generic.go @@ -35,7 +35,7 @@ func (gca *GenericCommandArgs) ServerDetails() (*config.ServerDetails, error) { } func (gc *GenericCommand) Run() (err error) { - if err = gc.preparePrerequisites("", false); err != nil { + if err = gc.PreparePrerequisites("", false); err != nil { return } return gc.runNpmGenericCommand() diff --git a/artifactory/commands/npm/installorci.go b/artifactory/commands/npm/installorci.go index ff3685a96..66920a886 100644 --- a/artifactory/commands/npm/installorci.go +++ b/artifactory/commands/npm/installorci.go @@ -82,21 +82,20 @@ func (nic *NpmInstallOrCiCommand) ServerDetails() (*config.ServerDetails, error) } func (nic *NpmInstallOrCiCommand) Run() (err error) { - if err = nic.preparePrerequisites(nic.repo, true); err != nil { + if err = nic.PreparePrerequisites(nic.repo, true); err != nil { return } - - if err = nic.prepareBuildInfoModule(); err != nil { - return - } - defer func() { e := nic.restoreNpmrcFunc() if err == nil { err = e } }() - if err = nic.createTempNpmrc(); err != nil { + if err = nic.CreateTempNpmrc(); err != nil { + return + } + + if err = nic.prepareBuildInfoModule(); err != nil { return } diff --git a/artifactory/commands/python/poetry.go b/artifactory/commands/python/poetry.go index eb9f4cd25..ad3ae9870 100644 --- a/artifactory/commands/python/poetry.go +++ b/artifactory/commands/python/poetry.go @@ -11,12 +11,10 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python/dependencies" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/spf13/viper" "io" - "net/url" "os" "os/exec" "path/filepath" @@ -138,37 +136,29 @@ func (pc *PoetryCommand) SetCommandName(commandName string) *PoetryCommand { } func (pc *PoetryCommand) SetPypiRepoUrlWithCredentials() error { - rtUrl, err := url.Parse(pc.serverDetails.GetArtifactoryUrl()) + rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(pc.serverDetails, pc.repository) if err != nil { - return errorutils.CheckError(err) - } - - username := pc.serverDetails.GetUser() - password := pc.serverDetails.GetPassword() - - // Get credentials from access-token if exists. - if pc.serverDetails.GetAccessToken() != "" { - if username == "" { - username = auth.ExtractUsernameFromAccessToken(pc.serverDetails.GetAccessToken()) - } - password = pc.serverDetails.GetAccessToken() + return err } - rtUrl.Path += "api/pypi/" + pc.repository + "/simple" if password != "" { - return pc.configPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password) + return ConfigPoetryRepo( + rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, + username, + password, + pc.poetryConfigRepoName) } return nil } -func (pc *PoetryCommand) configPoetryRepo(url, username, password string) error { +func ConfigPoetryRepo(url, username, password, configRepoName string) error { // Add the poetry repository config - err := runPoetryConfigCommand([]string{poetryConfigRepoPrefix + pc.poetryConfigRepoName, url}, false) + err := runPoetryConfigCommand([]string{poetryConfigRepoPrefix + configRepoName, url}, false) if err != nil { return err } // Set the poetry repository credentials - err = runPoetryConfigCommand([]string{poetryConfigAuthPrefix + pc.poetryConfigRepoName, username, password}, true) + err = runPoetryConfigCommand([]string{poetryConfigAuthPrefix + configRepoName, username, password}, true) if err != nil { return err } @@ -178,7 +168,7 @@ func (pc *PoetryCommand) configPoetryRepo(url, username, password string) error if err != nil { return errorutils.CheckError(err) } - return addRepoToPyprojectFile(filepath.Join(currentDir, pyproject), pc.poetryConfigRepoName, url) + return addRepoToPyprojectFile(filepath.Join(currentDir, pyproject), configRepoName, url) } func addRepoToPyprojectFile(filepath, poetryRepoName, repoUrl string) error { diff --git a/artifactory/commands/python/python.go b/artifactory/commands/python/python.go index 9f0a981c5..17c0e8911 100644 --- a/artifactory/commands/python/python.go +++ b/artifactory/commands/python/python.go @@ -18,6 +18,11 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) +const ( + PipenvRemoteRegistryFlag = "--pypi-mirror" + PipRemoteRegistryFlag = "-i" +) + type PythonCommand struct { serverDetails *config.ServerDetails pythonTool pythonutils.PythonTool @@ -103,32 +108,11 @@ func (pc *PythonCommand) SetCommandName(commandName string) *PythonCommand { } func (pc *PythonCommand) SetPypiRepoUrlWithCredentials() error { - rtUrl, err := url.Parse(pc.serverDetails.GetArtifactoryUrl()) + rtUrl, err := GetPypiRepoUrl(pc.serverDetails, pc.repository) if err != nil { - return errorutils.CheckError(err) - } - - username := pc.serverDetails.GetUser() - password := pc.serverDetails.GetPassword() - - // Get credentials from access-token if exists. - if pc.serverDetails.GetAccessToken() != "" { - if username == "" { - username = auth.ExtractUsernameFromAccessToken(pc.serverDetails.GetAccessToken()) - } - password = pc.serverDetails.GetAccessToken() - } - if password != "" { - rtUrl.User = url.UserPassword(username, password) - } - rtUrl.Path += "api/pypi/" + pc.repository + "/simple" - - if pc.pythonTool == pythonutils.Pip { - pc.args = append(pc.args, "-i") - } else if pc.pythonTool == pythonutils.Pipenv { - pc.args = append(pc.args, "--pypi-mirror") + return err } - pc.args = append(pc.args, rtUrl.String()) + pc.args = append(pc.args, GetPypiRemoteRegistryFlag(pc.pythonTool), rtUrl.String()) return nil } @@ -160,3 +144,41 @@ func (pc *PythonCommand) GetStdWriter() io.WriteCloser { func (pc *PythonCommand) GetErrWriter() io.WriteCloser { return nil } + +func GetPypiRepoUrl(serverDetails *config.ServerDetails, repository string) (*url.URL, error) { + rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(serverDetails, repository) + if err != nil { + return nil, err + } + if password != "" { + rtUrl.User = url.UserPassword(username, password) + } + return rtUrl, err +} + +func GetPypiRepoUrlWithCredentials(serverDetails *config.ServerDetails, repository string) (*url.URL, string, string, error) { + rtUrl, err := url.Parse(serverDetails.GetArtifactoryUrl()) + if err != nil { + return nil, "", "", errorutils.CheckError(err) + } + + username := serverDetails.GetUser() + password := serverDetails.GetPassword() + + // Get credentials from access-token if exists. + if serverDetails.GetAccessToken() != "" { + if username == "" { + username = auth.ExtractUsernameFromAccessToken(serverDetails.GetAccessToken()) + } + password = serverDetails.GetAccessToken() + } + rtUrl.Path += "api/pypi/" + repository + "/simple" + return rtUrl, username, password, err +} + +func GetPypiRemoteRegistryFlag(tool pythonutils.PythonTool) string { + if tool == pythonutils.Pip { + return PipRemoteRegistryFlag + } + return PipenvRemoteRegistryFlag +} diff --git a/artifactory/commands/yarn/yarn.go b/artifactory/commands/yarn/yarn.go index c171d046f..502a8306c 100644 --- a/artifactory/commands/yarn/yarn.go +++ b/artifactory/commands/yarn/yarn.go @@ -21,9 +21,14 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -const yarnrcFileName = ".yarnrc.yml" -const yarnrcBackupFileName = "jfrog.yarnrc.backup" -const npmScopesConfigName = "npmScopes" +const ( + YarnrcFileName = ".yarnrc.yml" + YarnrcBackupFileName = "jfrog.yarnrc.backup" + NpmScopesConfigName = "npmScopes" + yarnNpmRegistryServerEnv = "" + yarnNpmAuthIndent = "YARN_NPM_AUTH_IDENT" + yarnNpmAlwaysAuth = "YARN_NPM_ALWAYS_AUTH" +) type YarnCommand struct { executablePath string @@ -35,11 +40,9 @@ type YarnCommand struct { configFilePath string yarnArgs []string threads int - restoreYarnrcFunc func() error serverDetails *config.ServerDetails authArtDetails auth.ServiceDetails buildConfiguration *utils.BuildConfiguration - envVarsBackup map[string]*string buildInfoModule *build.YarnModule } @@ -92,18 +95,19 @@ func (yc *YarnCommand) Run() error { }() } - yc.restoreYarnrcFunc, err = commandUtils.BackupFile(filepath.Join(yc.workingDirectory, yarnrcFileName), filepath.Join(yc.workingDirectory, yarnrcBackupFileName)) + restoreYarnrcFunc, err := commandUtils.BackupFile(filepath.Join(yc.workingDirectory, YarnrcFileName), filepath.Join(yc.workingDirectory, YarnrcBackupFileName)) if err != nil { - return yc.restoreConfigurationsAndError(err) + return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } - if err = yc.modifyYarnConfigurations(); err != nil { - return yc.restoreConfigurationsAndError(err) + backupEnvMap, err := ModifyYarnConfigurations(yc.executablePath, yc.registry, yc.npmAuthIdent) + if err != nil { + return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } yc.buildInfoModule.SetArgs(filteredYarnArgs) if err = yc.buildInfoModule.Build(); err != nil { - return yc.restoreConfigurationsAndError(err) + return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } if yc.collectBuildInfo { @@ -111,7 +115,7 @@ func (yc *YarnCommand) Run() error { commandUtils.PrintMissingDependencies(missingDependencies) } - if err = yc.restoreConfigurationsFromBackup(); err != nil { + if err = RestoreConfigurationsFromBackup(backupEnvMap, restoreYarnrcFunc); err != nil { return err } @@ -161,6 +165,13 @@ func (yc *YarnCommand) readConfigFile() error { return err } +func RestoreConfigurationsAndError(envVarsBackup map[string]*string, restoreNpmrcFunc func() error, err error) error { + if restoreErr := RestoreConfigurationsFromBackup(envVarsBackup, restoreNpmrcFunc); restoreErr != nil { + return fmt.Errorf("two errors occurred:\n%s\n%s", restoreErr.Error(), err.Error()) + } + return err +} + func (yc *YarnCommand) preparePrerequisites() error { log.Debug("Preparing prerequisites.") var err error @@ -201,20 +212,31 @@ func (yc *YarnCommand) preparePrerequisites() error { yc.buildInfoModule.SetName(yc.buildConfiguration.GetModule()) } - if err = yc.setArtifactoryAuth(); err != nil { - return err + yc.registry, yc.npmAuthIdent, err = GetYarnAuthDetails(yc.serverDetails, yc.registry) + return err +} + +func (yc *YarnCommand) prepareBuildInfo() (missingDepsChan chan string, err error) { + log.Info("Preparing for dependencies information collection... For the first run of the build, the dependencies collection may take a few minutes. Subsequent runs should be faster.") + servicesManager, err := utils.CreateServiceManager(yc.serverDetails, -1, 0, false) + if err != nil { + return } - var npmAuthOutput string - npmAuthOutput, yc.registry, err = commandUtils.GetArtifactoryNpmRepoDetails(yc.repo, &yc.authArtDetails) + // Collect checksums from last build to decrease requests to Artifactory + buildName, err := yc.buildConfiguration.GetBuildName() if err != nil { - return err + return } - yc.npmAuthIdent, err = extractAuthIdentFromNpmAuth(npmAuthOutput) + previousBuildDependencies, err := commandUtils.GetDependenciesFromLatestBuild(servicesManager, buildName) if err != nil { - return err + return } - return nil + missingDepsChan = make(chan string) + collectChecksumsFunc := commandUtils.CreateCollectChecksumsFunc(previousBuildDependencies, servicesManager, missingDepsChan) + yc.buildInfoModule.SetTraverseDependenciesFunc(collectChecksumsFunc) + yc.buildInfoModule.SetThreads(yc.threads) + return } func (yc *YarnCommand) setYarnExecutable() error { @@ -228,34 +250,43 @@ func (yc *YarnCommand) setYarnExecutable() error { return nil } -func (yc *YarnCommand) setArtifactoryAuth() error { - authArtDetails, err := yc.serverDetails.CreateArtAuthConfig() +func GetYarnAuthDetails(server *config.ServerDetails, repo string) (string, string, error) { + authRtDetails, err := setArtifactoryAuth(server) if err != nil { - return err + return "", "", err } - if authArtDetails.GetSshAuthHeaders() != nil { - return errorutils.CheckErrorf("SSH authentication is not supported in this command") + var npmAuthOutput string + npmAuthOutput, registry, err := commandUtils.GetArtifactoryNpmRepoDetails(repo, &authRtDetails) + if err != nil { + return "", "", err } - yc.authArtDetails = authArtDetails - return nil + npmAuthIdent, err := extractAuthIdentFromNpmAuth(npmAuthOutput) + if err != nil { + return "", "", err + } + return registry, npmAuthIdent, nil } -func (yc *YarnCommand) restoreConfigurationsFromBackup() error { - if err := yc.restoreEnvironmentVariables(); err != nil { - return err +func setArtifactoryAuth(server *config.ServerDetails) (auth.ServiceDetails, error) { + authArtDetails, err := server.CreateArtAuthConfig() + if err != nil { + return nil, err + } + if authArtDetails.GetSshAuthHeaders() != nil { + return nil, errorutils.CheckErrorf("SSH authentication is not supported in this command") } - return yc.restoreYarnrcFunc() + return authArtDetails, nil } -func (yc *YarnCommand) restoreConfigurationsAndError(err error) error { - if restoreErr := yc.restoreConfigurationsFromBackup(); restoreErr != nil { - return fmt.Errorf("two errors occurred:\n%s\n%s", restoreErr.Error(), err.Error()) +func RestoreConfigurationsFromBackup(envVarsBackup map[string]*string, restoreYarnrcFunc func() error) error { + if err := restoreEnvironmentVariables(envVarsBackup); err != nil { + return err } - return err + return restoreYarnrcFunc() } -func (yc *YarnCommand) restoreEnvironmentVariables() error { - for key, value := range yc.envVarsBackup { +func restoreEnvironmentVariables(envVarsBackup map[string]*string) error { + for key, value := range envVarsBackup { if value == nil { if err := os.Unsetenv(key); err != nil { return err @@ -270,23 +301,26 @@ func (yc *YarnCommand) restoreEnvironmentVariables() error { return nil } -func (yc *YarnCommand) modifyYarnConfigurations() error { - yc.envVarsBackup = make(map[string]*string) - - if err := yc.backupAndSetEnvironmentVariable("YARN_NPM_REGISTRY_SERVER", yc.registry); err != nil { - return err - } - - if err := yc.backupAndSetEnvironmentVariable("YARN_NPM_AUTH_IDENT", yc.npmAuthIdent); err != nil { - return err +func ModifyYarnConfigurations(execPath, registry, npmAuthIdent string) (map[string]*string, error) { + envVarsUpdated := map[string]string{ + yarnNpmRegistryServerEnv: registry, + yarnNpmAuthIndent: npmAuthIdent, + yarnNpmAlwaysAuth: "true", } - - if err := yc.backupAndSetEnvironmentVariable("YARN_NPM_ALWAYS_AUTH", "true"); err != nil { - return err + envVarsBackup := make(map[string]*string) + for key, value := range envVarsUpdated { + oldVal, err := backupAndSetEnvironmentVariable(key, value) + if err != nil { + return nil, err + } + envVarsBackup[key] = &oldVal } - // Update scoped registries (these cannot be set in environment variables) - npmScopesStr, err := yarn.ConfigGet(npmScopesConfigName, yc.executablePath, true) + return envVarsBackup, errorutils.CheckError(UpdateScopeRegistries(execPath, registry, npmAuthIdent)) +} + +func UpdateScopeRegistries(execPath, registry, npmAuthIdent string) error { + npmScopesStr, err := yarn.ConfigGet(NpmScopesConfigName, execPath, true) if err != nil { return err } @@ -295,7 +329,7 @@ func (yc *YarnCommand) modifyYarnConfigurations() error { if err != nil { return errorutils.CheckError(err) } - artifactoryScope := yarnNpmScope{NpmAlwaysAuth: true, NpmAuthIdent: yc.npmAuthIdent, NpmRegistryServer: yc.registry} + artifactoryScope := yarnNpmScope{NpmAlwaysAuth: true, NpmAuthIdent: npmAuthIdent, NpmRegistryServer: registry} for scopeName := range npmScopesMap { npmScopesMap[scopeName] = artifactoryScope } @@ -303,8 +337,7 @@ func (yc *YarnCommand) modifyYarnConfigurations() error { if err != nil { return errorutils.CheckError(err) } - err = yarn.ConfigSet(npmScopesConfigName, string(updatedNpmScopesStr), yc.executablePath, true) - return errorutils.CheckError(err) + return yarn.ConfigSet(NpmScopesConfigName, string(updatedNpmScopesStr), execPath, true) } type yarnNpmScope struct { @@ -313,38 +346,9 @@ type yarnNpmScope struct { NpmRegistryServer string `json:"npmRegistryServer,omitempty"` } -func (yc *YarnCommand) backupAndSetEnvironmentVariable(key, value string) error { - oldVal, exist := os.LookupEnv(key) - if exist { - yc.envVarsBackup[key] = &oldVal - } else { - yc.envVarsBackup[key] = nil - } - - return errorutils.CheckError(os.Setenv(key, value)) -} - -func (yc *YarnCommand) prepareBuildInfo() (missingDepsChan chan string, err error) { - log.Info("Preparing for dependencies information collection... For the first run of the build, the dependencies collection may take a few minutes. Subsequent runs should be faster.") - servicesManager, err := utils.CreateServiceManager(yc.serverDetails, -1, 0, false) - if err != nil { - return - } - - // Collect checksums from last build to decrease requests to Artifactory - buildName, err := yc.buildConfiguration.GetBuildName() - if err != nil { - return - } - previousBuildDependencies, err := commandUtils.GetDependenciesFromLatestBuild(servicesManager, buildName) - if err != nil { - return - } - missingDepsChan = make(chan string) - collectChecksumsFunc := commandUtils.CreateCollectChecksumsFunc(previousBuildDependencies, servicesManager, missingDepsChan) - yc.buildInfoModule.SetTraverseDependenciesFunc(collectChecksumsFunc) - yc.buildInfoModule.SetThreads(yc.threads) - return +func backupAndSetEnvironmentVariable(key, value string) (string, error) { + oldVal, _ := os.LookupEnv(key) + return oldVal, errorutils.CheckError(os.Setenv(key, value)) } // npmAuth we get back from Artifactory includes several fields, but we need only the field '_auth' diff --git a/artifactory/utils/buildinfoproperties.go b/artifactory/utils/buildinfoproperties.go index 1e24778a6..877588f54 100644 --- a/artifactory/utils/buildinfoproperties.go +++ b/artifactory/utils/buildinfoproperties.go @@ -155,36 +155,31 @@ func ReadConfigFile(configPath string, configType ConfigType) (config *viper.Vip return config, errorutils.CheckError(err) } -func ReadGradleConfig(path string, useWrapperIfMissingConfig bool) (config *viper.Viper, err error) { +func ReadGradleConfig(path string, gradleConfigParams map[string]any) (config *viper.Viper, err error) { if path == "" { - config = createDefaultGradleConfig(useWrapperIfMissingConfig) + config = createDefaultConfigWithParams(YAML, Gradle.String(), gradleConfigParams) } else { config, err = ReadConfigFile(path, YAML) } return } -func ReadMavenConfig(path string) (config *viper.Viper, err error) { +func ReadMavenConfig(path string, mvnProps map[string]any) (config *viper.Viper, err error) { if path == "" { - config = createDefaultMavenConfig() + config = createDefaultConfigWithParams(YAML, Maven.String(), mvnProps) } else { config, err = ReadConfigFile(path, YAML) } return } -func createDefaultMavenConfig() *viper.Viper { +func createDefaultConfigWithParams(configType ConfigType, technology string, params map[string]any) *viper.Viper { vConfig := viper.New() - vConfig.SetConfigType(string(YAML)) - vConfig.Set("type", Maven.String()) - return vConfig -} - -func createDefaultGradleConfig(useWrapperIfMissingConfig bool) *viper.Viper { - vConfig := viper.New() - vConfig.SetConfigType(string(YAML)) - vConfig.Set("type", Gradle.String()) - vConfig.Set("usewrapper", useWrapperIfMissingConfig) + vConfig.SetConfigType(string(configType)) + vConfig.Set("type", technology) + for key, value := range params { + vConfig.Set(key, value) + } return vConfig } diff --git a/utils/mvn/utils.go b/utils/mvn/utils.go index 79c0a377d..95a4a3921 100644 --- a/utils/mvn/utils.go +++ b/utils/mvn/utils.go @@ -14,7 +14,8 @@ import ( "github.com/spf13/viper" ) -func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration, goals []string, threads int, insecureTls, disableDeploy bool) error { +func RunMvn(vConfig *viper.Viper, buildArtifactsDetailsFile string, buildConf *utils.BuildConfiguration, + goals []string, threads int, insecureTls, disableDeploy bool) error { buildInfoService := utils.CreateBuildInfoService() buildName, err := buildConf.GetBuildName() if err != nil { diff --git a/xray/audit/go/golang.go b/xray/audit/go/golang.go index 67949e0e2..a760fa78e 100644 --- a/xray/audit/go/golang.go +++ b/xray/audit/go/golang.go @@ -2,6 +2,9 @@ package _go import ( "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/golang" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "os" "strings" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -14,11 +17,16 @@ const ( goSourceCodePrefix = "github.com/golang/go:v" ) -func BuildDependencyTree() (dependencyTree []*services.GraphNode, err error) { +func BuildDependencyTree(server *config.ServerDetails, remoteGoRepo string) (dependencyTree []*services.GraphNode, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return } + if remoteGoRepo != "" { + if err = setGoProxy(server, remoteGoRepo); err != nil { + return + } + } // Calculate go dependencies graph dependenciesGraph, err := goutils.GetDependenciesGraph(currentDir) if err != nil { @@ -51,6 +59,15 @@ func BuildDependencyTree() (dependencyTree []*services.GraphNode, err error) { return } +func setGoProxy(server *config.ServerDetails, remoteGoRepo string) error { + repoUrl, err := golang.GetGoRepoUrl(server, remoteGoRepo) + if err != nil { + return err + } + repoUrl += "|direct" + return os.Setenv("GOPROXY", repoUrl) +} + func populateGoDependencyTree(currNode *services.GraphNode, dependenciesGraph map[string][]string, dependenciesList map[string]bool) { if currNode.NodeHasLoop() { return diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index b4721b79a..079838e80 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -12,11 +12,11 @@ import ( const gradlew = "gradlew" -func BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile bool) (dependencyTree []*services.GraphNode, err error) { +func BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (dependencyTree []*services.GraphNode, err error) { buildConfiguration, cleanBuild := createBuildConfiguration("audit-gradle") defer cleanBuild(err) - err = runGradle(buildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile) + err = runGradle(buildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile, gradleConfigParams) if err != nil { return } @@ -25,7 +25,7 @@ func BuildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile boo return } -func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile bool) (err error) { +func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (err error) { tasks := "clean compileJava " if !excludeTestDeps { tasks += "compileTestJava " @@ -49,9 +49,10 @@ func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, us if err != nil { return } + gradleConfigParams["usewrapper"] = useWrapper } // Read config - vConfig, err := utils.ReadGradleConfig(configFilePath, useWrapper) + vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams) if err != nil { return err } diff --git a/xray/audit/java/javautils.go b/xray/audit/java/javautils.go index 509b5d68f..b657ef8fa 100644 --- a/xray/audit/java/javautils.go +++ b/xray/audit/java/javautils.go @@ -1,6 +1,7 @@ package java import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "strconv" "time" @@ -15,6 +16,15 @@ const ( GavPackageTypeIdentifier = "gav://" ) +type DependencyTreeParams struct { + Tool coreutils.Technology + InsecureTls bool + IgnoreConfigFile bool + ExcludeTestDeps bool + UseWrapper bool + JavaProps map[string]any +} + func createBuildConfiguration(buildName string) (*artifactoryUtils.BuildConfiguration, func(err error)) { buildConfiguration := artifactoryUtils.NewBuildConfiguration(buildName, strconv.FormatInt(time.Now().Unix(), 10), "", "") return buildConfiguration, func(err error) { @@ -124,6 +134,13 @@ func hasLoop(idsAdded []string, idToAdd string) bool { return false } +func BuildDependencyTree(params *DependencyTreeParams) (modules []*services.GraphNode, err error) { + if params.Tool == coreutils.Maven { + return BuildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.JavaProps) + } + return BuildGradleDependencyTree(params.ExcludeTestDeps, params.UseWrapper, params.IgnoreConfigFile, params.JavaProps) +} + type dependencyMultimap struct { multimap map[string]map[string]*buildinfo.Dependency } diff --git a/xray/audit/java/mvn.go b/xray/audit/java/mvn.go index 2c6706ea9..c544feeb6 100644 --- a/xray/audit/java/mvn.go +++ b/xray/audit/java/mvn.go @@ -9,11 +9,11 @@ import ( "github.com/jfrog/jfrog-client-go/xray/services" ) -func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool) (modules []*services.GraphNode, err error) { +func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool, mvnProps map[string]any) (modules []*services.GraphNode, err error) { buildConfiguration, cleanBuild := createBuildConfiguration("audit-mvn") defer cleanBuild(err) - err = runMvn(buildConfiguration, insecureTls, ignoreConfigFile) + err = runMvn(buildConfiguration, insecureTls, ignoreConfigFile, mvnProps) if err != nil { return } @@ -21,7 +21,7 @@ func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool) (modules []*serv return createGavDependencyTree(buildConfiguration) } -func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile bool) (err error) { +func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile bool, mvnProps map[string]any) (err error) { goals := []string{"-B", "compile", "test-compile"} log.Debug(fmt.Sprintf("mvn command goals: %v", goals)) configFilePath := "" @@ -36,7 +36,7 @@ func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreCon } } // Read config - vConfig, err := utils.ReadMavenConfig(configFilePath) + vConfig, err := utils.ReadMavenConfig(configFilePath, mvnProps) if err != nil { return err } diff --git a/xray/audit/java/mvn_test.go b/xray/audit/java/mvn_test.go index 15e8a47cc..ced00dd70 100644 --- a/xray/audit/java/mvn_test.go +++ b/xray/audit/java/mvn_test.go @@ -14,7 +14,7 @@ func TestMavenTreesMultiModule(t *testing.T) { defer cleanUp() // Run getModulesDependencyTrees - modulesDependencyTrees, err := BuildMvnDependencyTree(false, true) + modulesDependencyTrees, err := BuildMvnDependencyTree(false, true, nil) if assert.NoError(t, err) && assert.NotEmpty(t, modulesDependencyTrees) { // Check root module multi := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test:multi:3.7-SNAPSHOT") diff --git a/xray/audit/python/python.go b/xray/audit/python/python.go index 604ff7e5b..4dc83ab10 100644 --- a/xray/audit/python/python.go +++ b/xray/audit/python/python.go @@ -3,6 +3,7 @@ package python import ( "fmt" "github.com/jfrog/build-info-go/utils/pythonutils" + rtpython "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/xray/audit" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -20,8 +21,15 @@ const ( pythonPackageTypeIdentifier = "pypi://" ) -func BuildDependencyTree(pythonTool pythonutils.PythonTool, requirementsFile string) (dependencyTree []*services.GraphNode, err error) { - dependenciesGraph, directDependenciesList, err := getDependencies(pythonTool, requirementsFile) +type AuditPython struct { + Server *config.ServerDetails + Tool pythonutils.PythonTool + RemotePypiRepo string + PipRequirementsFile string +} + +func BuildDependencyTree(auditPython *AuditPython) (dependencyTree []*services.GraphNode, err error) { + dependenciesGraph, directDependenciesList, err := getDependencies(auditPython) if err != nil { return } @@ -41,7 +49,7 @@ func BuildDependencyTree(pythonTool pythonutils.PythonTool, requirementsFile str return []*services.GraphNode{root}, nil } -func getDependencies(pythonTool pythonutils.PythonTool, requirementsFile string) (dependenciesGraph map[string][]string, directDependencies []string, err error) { +func getDependencies(auditPython *AuditPython) (dependenciesGraph map[string][]string, directDependencies []string, err error) { wd, err := os.Getwd() if errorutils.CheckError(err) != nil { return @@ -75,7 +83,7 @@ func getDependencies(pythonTool pythonutils.PythonTool, requirementsFile string) return } - restoreEnv, err := runPythonInstall(pythonTool, requirementsFile) + restoreEnv, err := runPythonInstall(auditPython) defer func() { e := restoreEnv() if err == nil { @@ -90,54 +98,82 @@ func getDependencies(pythonTool pythonutils.PythonTool, requirementsFile string) if err != nil { return } - dependenciesGraph, directDependencies, err = pythonutils.GetPythonDependencies(pythonTool, tempDirPath, localDependenciesPath) + dependenciesGraph, directDependencies, err = pythonutils.GetPythonDependencies(auditPython.Tool, tempDirPath, localDependenciesPath) if err != nil { if _, innerErr := audit.GetExecutableVersion("python"); innerErr != nil { log.Error(innerErr) } - if _, innerErr := audit.GetExecutableVersion(string(pythonTool)); innerErr != nil { + if _, innerErr := audit.GetExecutableVersion(string(auditPython.Tool)); innerErr != nil { log.Error(innerErr) } } return } -func runPythonInstall(pythonTool pythonutils.PythonTool, requirementsFile string) (restoreEnv func() error, err error) { +func runPythonInstall(auditPython *AuditPython) (restoreEnv func() error, err error) { + switch auditPython.Tool { + case pythonutils.Pip: + return installPipDeps(auditPython) + case pythonutils.Pipenv: + return installPipenvDeps(auditPython) + case pythonutils.Poetry: + return installPoetryDeps(auditPython) + } + return +} + +func installPoetryDeps(auditPython *AuditPython) (restoreEnv func() error, err error) { restoreEnv = func() error { return nil } - switch pythonTool { - case pythonutils.Pip: - restoreEnv, err = SetPipVirtualEnvPath() - if err != nil { - return - } - err = runPipInstall(requirementsFile) - if err != nil && requirementsFile == "" { - log.Debug(err.Error() + "\ntrying to install using a requirements file.") - reqErr := runPipInstall("requirements.txt") - if reqErr != nil { - // Return Pip install error and log the requirements fallback error. - log.Debug(reqErr.Error()) - } else { - err = nil + if auditPython.RemotePypiRepo != "" { + rtUrl, username, password, err := rtpython.GetPypiRepoUrlWithCredentials(auditPython.Server, auditPython.RemotePypiRepo) + if password != "" { + err = rtpython.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, auditPython.RemotePypiRepo) + if err != nil { + return restoreEnv, err } } - case pythonutils.Pipenv: - // Set virtualenv path to venv dir - err = os.Setenv("WORKON_HOME", ".jfrog") - if err != nil { - return - } - restoreEnv = func() error { - return os.Unsetenv("WORKON_HOME") - } - // Run 'pipenv install -d' - err = executeCommand("pipenv", "install", "-d") + } + // Run 'poetry install' + return restoreEnv, executeCommand("poetry", "install") +} - case pythonutils.Poetry: - // Run 'poetry install' - err = executeCommand("poetry", "install") +func installPipenvDeps(auditPython *AuditPython) (restoreEnv func() error, err error) { + // Set virtualenv path to venv dir + err = os.Setenv("WORKON_HOME", ".jfrog") + if err != nil { + return + } + restoreEnv = func() error { + return os.Unsetenv("WORKON_HOME") + } + if auditPython.RemotePypiRepo != "" { + return restoreEnv, runPipenvInstallFromRemoteRegistry(auditPython.Server, auditPython.RemotePypiRepo) + } + // Run 'pipenv install -d' + return restoreEnv, executeCommand("pipenv", "install", "-d") +} + +func installPipDeps(auditPython *AuditPython) (restoreEnv func() error, err error) { + restoreEnv, err = SetPipVirtualEnvPath() + if err != nil { + return + } + if auditPython.RemotePypiRepo != "" { + return restoreEnv, runPipInstallFromRemoteRegistry(auditPython.Server, auditPython.RemotePypiRepo, auditPython.PipRequirementsFile) + } + pipInstallArgs := getPipInstallArgs(auditPython.PipRequirementsFile) + err = runPipInstall(pipInstallArgs...) + if err != nil && auditPython.PipRequirementsFile == "" { + log.Debug(err.Error() + "\ntrying to install using a requirements file.") + reqErr := runPipInstall("requirements.txt") + if reqErr != nil { + // Return Pip install error and log the requirements fallback error. + log.Debug(reqErr.Error()) + } else { + err = nil + } } return } @@ -155,19 +191,46 @@ func executeCommand(executable string, args ...string) error { return nil } -func runPipInstall(requirementsFile string) error { +func runPipInstall(args ...string) error { // Try getting 'pip3' executable, if not found use 'pip' + pipExec := getPipExec() + return executeCommand(pipExec, args...) +} + +func getPipInstallArgs(requirementsFile string) []string { + if requirementsFile == "" { + // Run 'pip install .' + return []string{"install", "."} + } + // Run pip 'install -r requirements ' + return []string{"install", "-r", requirementsFile} +} + +func getPipExec() string { pipExec, _ := exec.LookPath("pip3") if pipExec == "" { pipExec = "pip" } - if requirementsFile == "" { - // Run 'pip install .' - return executeCommand(pipExec, "install", ".") - } else { - // Run pip 'install -r requirements ' - return executeCommand(pipExec, "install", "-r", requirementsFile) + return pipExec +} + +func runPipInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName, pipRequirementsFile string) (err error) { + rtUrl, err := rtpython.GetPypiRepoUrl(server, depsRepoName) + if err != nil { + return err + } + args := getPipInstallArgs(pipRequirementsFile) + args = append(args, rtpython.GetPypiRemoteRegistryFlag(pythonutils.Pip), rtUrl.String()) + return runPipInstall(args...) +} + +func runPipenvInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName string) (err error) { + rtUrl, err := rtpython.GetPypiRepoUrl(server, depsRepoName) + if err != nil { + return err } + args := []string{"install", "-d", rtpython.GetPypiRemoteRegistryFlag(pythonutils.Pipenv), rtUrl.String()} + return executeCommand("pipenv", args...) } // Execute virtualenv command: "virtualenv venvdir" / "python3 -m venv venvdir" and set path diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 9155df0f4..ff77eb96e 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -3,16 +3,17 @@ package audit import ( "errors" "fmt" - "github.com/jfrog/jfrog-cli-core/v2/xray/audit" + "github.com/jfrog/jfrog-cli-core/v2/xray/audit/java" "os" "path/filepath" "strings" + "github.com/jfrog/jfrog-cli-core/v2/xray/audit" + "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" _go "github.com/jfrog/jfrog-cli-core/v2/xray/audit/go" - "github.com/jfrog/jfrog-cli-core/v2/xray/audit/java" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/npm" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/nuget" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/python" @@ -31,16 +32,70 @@ type Params struct { excludeTestDeps bool insecureTls bool useWrapper bool + depsRepo string requirementsFile string technologies []string workingDirs []string args []string + installFunc func(tech string) error } func NewAuditParams() *Params { return &Params{} } +func (params *Params) InstallFunc() func(tech string) error { + return params.installFunc +} + +func (params *Params) XrayGraphScanParams() services.XrayGraphScanParams { + return params.xrayGraphScanParams +} + +func (params *Params) ServerDetails() *config.ServerDetails { + return params.serverDetails +} + +func (params *Params) Progress() ioUtils.ProgressMgr { + return params.progress +} + +func (params *Params) IgnoreConfigFile() bool { + return params.ignoreConfigFile +} + +func (params *Params) ExcludeTestDeps() bool { + return params.excludeTestDeps +} + +func (params *Params) InsecureTls() bool { + return params.insecureTls +} + +func (params *Params) UseWrapper() bool { + return params.useWrapper +} + +func (params *Params) DepsRepo() string { + return params.depsRepo +} + +func (params *Params) RequirementsFile() string { + return params.requirementsFile +} + +func (params *Params) Technologies() []string { + return params.technologies +} + +func (params *Params) WorkingDirs() []string { + return params.workingDirs +} + +func (params *Params) Args() []string { + return params.args +} + func (params *Params) SetXrayGraphScanParams(xrayGraphScanParams services.XrayGraphScanParams) *Params { params.xrayGraphScanParams = xrayGraphScanParams return params @@ -96,6 +151,16 @@ func (params *Params) SetTechnologies(technologies ...string) *Params { return params } +func (params *Params) SetDepsRepo(depsRepo string) *Params { + params.depsRepo = depsRepo + return params +} + +func (params *Params) SetInstallFunc(installFunc func(tech string) error) *Params { + params.installFunc = installFunc + return params +} + // GenericAudit audits all the projects found in the given workingDirs func GenericAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bool, err error) { if len(params.workingDirs) == 0 { @@ -187,18 +252,20 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc params.progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) } switch tech { - case coreutils.Maven: - dependencyTrees, e = java.BuildMvnDependencyTree(params.insecureTls, params.ignoreConfigFile) - case coreutils.Gradle: - dependencyTrees, e = java.BuildGradleDependencyTree(params.excludeTestDeps, params.useWrapper, params.ignoreConfigFile) + case coreutils.Maven, coreutils.Gradle: + dependencyTrees, e = getJavaDependencyTree(params, tech) case coreutils.Npm: dependencyTrees, e = npm.BuildDependencyTree(params.args) case coreutils.Yarn: dependencyTrees, e = yarn.BuildDependencyTree() case coreutils.Go: - dependencyTrees, e = _go.BuildDependencyTree() + dependencyTrees, e = _go.BuildDependencyTree(params.serverDetails, params.depsRepo) case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: - dependencyTrees, e = python.BuildDependencyTree(pythonutils.PythonTool(tech), params.requirementsFile) + dependencyTrees, e = python.BuildDependencyTree(&python.AuditPython{ + Server: params.serverDetails, + Tool: pythonutils.PythonTool(tech), + RemotePypiRepo: params.depsRepo, + PipRequirementsFile: params.requirementsFile}) case coreutils.Nuget: dependencyTrees, e = nuget.BuildDependencyTree() default: @@ -208,6 +275,37 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc return dependencyTrees, e } +func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*services.GraphNode, error) { + var javaProps map[string]any + if params.DepsRepo() != "" { + javaProps = createJavaProps(params.DepsRepo(), params.ServerDetails()) + } + return java.BuildDependencyTree(&java.DependencyTreeParams{ + Tool: tech, + InsecureTls: params.insecureTls, + IgnoreConfigFile: params.ignoreConfigFile, + ExcludeTestDeps: params.excludeTestDeps, + UseWrapper: params.useWrapper, + JavaProps: javaProps, + }) +} + +func createJavaProps(depsRepo string, serverDetails *config.ServerDetails) map[string]any { + authPass := serverDetails.Password + if serverDetails.AccessToken != "" { + authPass = serverDetails.AccessToken + } + return map[string]any{ + "resolver.username": serverDetails.User, + "resolver.password": authPass, + "resolver.url": serverDetails.ArtifactoryUrl, + "resolver.releaseRepo": depsRepo, + "resolver.repo": depsRepo, + "resolver.snapshotRepo": depsRepo, + "buildInfoConfig.artifactoryResolutionEnabled": true, + } +} + func detectedTechnologies() (technologies []string, err error) { wd, err := os.Getwd() if errorutils.CheckError(err) != nil { diff --git a/xray/commands/testdata/poetry-project/pyproject.toml b/xray/commands/testdata/poetry-project/pyproject.toml index 52ea7daf4..045c50e5d 100644 --- a/xray/commands/testdata/poetry-project/pyproject.toml +++ b/xray/commands/testdata/poetry-project/pyproject.toml @@ -1,14 +1,14 @@ -[tool.poetry] +[Tool.poetry] name = "my-poetry-project" version = "0.1.0" description = "" authors = ["Tal Arian "] -[tool.poetry.dependencies] +[Tool.poetry.dependencies] python = "^3.8" numpy = "^1.23.0" -[tool.poetry.dev-dependencies] +[Tool.poetry.dev-dependencies] pytest = "^5.2" [build-system] From 1f883b071ba405da11dc4f2d8be6abb3bdf10d04 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 14 Feb 2023 08:21:43 +0200 Subject: [PATCH 04/48] fix tests --- artifactory/commands/yarn/yarn.go | 3 +-- artifactory/commands/yarn/yarn_test.go | 14 +++++++------- xray/audit/go/gloang_test.go | 11 ++++++++++- xray/audit/java/gradle_test.go | 6 +++--- xray/audit/python/python.go | 3 +++ xray/audit/python/python_test.go | 10 +++++----- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/artifactory/commands/yarn/yarn.go b/artifactory/commands/yarn/yarn.go index 502a8306c..1b937f1ee 100644 --- a/artifactory/commands/yarn/yarn.go +++ b/artifactory/commands/yarn/yarn.go @@ -41,7 +41,6 @@ type YarnCommand struct { yarnArgs []string threads int serverDetails *config.ServerDetails - authArtDetails auth.ServiceDetails buildConfiguration *utils.BuildConfiguration buildInfoModule *build.YarnModule } @@ -287,7 +286,7 @@ func RestoreConfigurationsFromBackup(envVarsBackup map[string]*string, restoreYa func restoreEnvironmentVariables(envVarsBackup map[string]*string) error { for key, value := range envVarsBackup { - if value == nil { + if value == nil || *value == "" { if err := os.Unsetenv(key); err != nil { return err } diff --git a/artifactory/commands/yarn/yarn_test.go b/artifactory/commands/yarn/yarn_test.go index cd47cc8e3..296ab2440 100644 --- a/artifactory/commands/yarn/yarn_test.go +++ b/artifactory/commands/yarn/yarn_test.go @@ -33,23 +33,23 @@ func TestValidateSupportedCommand(t *testing.T) { func TestSetAndRestoreEnvironmentVariables(t *testing.T) { const jfrogCliTestingEnvVar = "JFROG_CLI_ENV_VAR_FOR_TESTING" - yarnCmd := &YarnCommand{envVarsBackup: make(map[string]*string)} - // Check backup and restore of an existing variable setEnvCallback := tests.SetEnvWithCallbackAndAssert(t, jfrogCliTestingEnvVar, "abc") - err := yarnCmd.backupAndSetEnvironmentVariable(jfrogCliTestingEnvVar, "new-value") + backupEnvsMap := make(map[string]*string) + oldVal, err := backupAndSetEnvironmentVariable(jfrogCliTestingEnvVar, "new-value") assert.NoError(t, err) assert.Equal(t, "new-value", os.Getenv(jfrogCliTestingEnvVar)) - err = yarnCmd.restoreEnvironmentVariables() - assert.NoError(t, err) + backupEnvsMap[jfrogCliTestingEnvVar] = &oldVal + assert.NoError(t, restoreEnvironmentVariables(backupEnvsMap)) assert.Equal(t, "abc", os.Getenv(jfrogCliTestingEnvVar)) // Check backup and restore of a variable that doesn't exist setEnvCallback() - err = yarnCmd.backupAndSetEnvironmentVariable(jfrogCliTestingEnvVar, "another-value") + oldVal, err = backupAndSetEnvironmentVariable(jfrogCliTestingEnvVar, "another-value") assert.NoError(t, err) assert.Equal(t, "another-value", os.Getenv(jfrogCliTestingEnvVar)) - err = yarnCmd.restoreEnvironmentVariables() + backupEnvsMap[jfrogCliTestingEnvVar] = &oldVal + err = restoreEnvironmentVariables(backupEnvsMap) assert.NoError(t, err) _, exist := os.LookupEnv(jfrogCliTestingEnvVar) assert.False(t, exist) diff --git a/xray/audit/go/gloang_test.go b/xray/audit/go/gloang_test.go index 69b3ff3d7..85fe26e9a 100644 --- a/xray/audit/go/gloang_test.go +++ b/xray/audit/go/gloang_test.go @@ -2,6 +2,8 @@ package _go import ( "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "os" "strings" "testing" @@ -24,8 +26,15 @@ func TestBuildGoDependencyList(t *testing.T) { assert.NoError(t, err) // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree() + server := &config.ServerDetails{ + Url: "https://api.go.here", + ArtifactoryUrl: "https://api.go.here/artifactory", + User: "user", + AccessToken: "sdsdccs2232", + } + rootNode, err := BuildDependencyTree(server, "test-remote") assert.NoError(t, err) + assert.Equal(t, "https://user:sdsdccs2232@api.go.here/artifactoryapi/go/test-remote|direct", os.Getenv("GOPROXY")) assert.NotEmpty(t, rootNode) // Check root module diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 3f36cf63a..1701d3877 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -17,7 +17,7 @@ func TestGradleTreesWithoutConfig(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := BuildGradleDependencyTree(false, true, true) + modulesDependencyTrees, err := BuildGradleDependencyTree(false, true, true, nil) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { assert.Len(t, modulesDependencyTrees, 5) // Check module @@ -40,7 +40,7 @@ func TestGradleTreesWithConfig(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := BuildGradleDependencyTree(false, false, false) + modulesDependencyTrees, err := BuildGradleDependencyTree(false, false, false, nil) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { assert.Len(t, modulesDependencyTrees, 3) @@ -64,7 +64,7 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := BuildGradleDependencyTree(true, true, true) + modulesDependencyTrees, err := BuildGradleDependencyTree(true, true, true, nil) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { assert.Len(t, modulesDependencyTrees, 5) // Check module diff --git a/xray/audit/python/python.go b/xray/audit/python/python.go index 4dc83ab10..35a44f57d 100644 --- a/xray/audit/python/python.go +++ b/xray/audit/python/python.go @@ -128,6 +128,9 @@ func installPoetryDeps(auditPython *AuditPython) (restoreEnv func() error, err e } if auditPython.RemotePypiRepo != "" { rtUrl, username, password, err := rtpython.GetPypiRepoUrlWithCredentials(auditPython.Server, auditPython.RemotePypiRepo) + if err != nil { + return restoreEnv, err + } if password != "" { err = rtpython.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, auditPython.RemotePypiRepo) if err != nil { diff --git a/xray/audit/python/python_test.go b/xray/audit/python/python_test.go index b24d2b4bb..de378cf07 100644 --- a/xray/audit/python/python_test.go +++ b/xray/audit/python/python_test.go @@ -39,7 +39,7 @@ func testBuildPipDependencyListSetuppy(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "setuppyproject")) defer cleanUp() // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(pythonutils.Pip, "") + rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) assert.NoError(t, err) assert.Len(t, rootNode, 1) if len(rootNode) > 0 { @@ -60,7 +60,7 @@ func TestPipDependencyListRequirementsFallback(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "requirementsproject")) defer cleanUp() // No requirements file field specified, expect the command to use the fallback 'pip install -r requirements.txt' command - rootNode, err := BuildDependencyTree(pythonutils.Pip, "") + rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) assert.NoError(t, err) assert.Len(t, rootNode, 1) if assert.True(t, len(rootNode[0].Nodes) > 2) { @@ -77,7 +77,7 @@ func TestBuildPipDependencyListRequirements(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "requirementsproject")) defer cleanUp() // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(pythonutils.Pip, "requirements.txt") + rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip, PipRequirementsFile: "requirements.txt"}) assert.NoError(t, err) assert.Len(t, rootNode, 1) if len(rootNode) > 0 { @@ -96,7 +96,7 @@ func TestBuildPipenvDependencyList(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, "pipenv-project") defer cleanUp() // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(pythonutils.Pipenv, "") + rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pipenv}) if err != nil { t.Fatal(err) } @@ -117,7 +117,7 @@ func TestBuildPoetryDependencyList(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, "poetry-project") defer cleanUp() // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(pythonutils.Poetry, "") + rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Poetry}) if err != nil { t.Fatal(err) } From e1220832a9cc39cc8cbd73106ee3fb77ec4ba7c9 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 14 Feb 2023 09:19:14 +0200 Subject: [PATCH 05/48] Fix tests --- .../commands/dotnet/dotnetcommand_test.go | 54 +++++++++++++++++++ xray/audit/java/gradle.go | 4 +- xray/audit/python/python_test.go | 2 +- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/artifactory/commands/dotnet/dotnetcommand_test.go b/artifactory/commands/dotnet/dotnetcommand_test.go index 248ed5f2b..f6e04d636 100644 --- a/artifactory/commands/dotnet/dotnetcommand_test.go +++ b/artifactory/commands/dotnet/dotnetcommand_test.go @@ -60,6 +60,60 @@ func TestGetFlagValueExists(t *testing.T) { } } +func TestInitNewConfig(t *testing.T) { + tmpDir, err := fileutils.CreateTempDir() + defer func() { + assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) + }() + assert.NoError(t, err) + repoName := "test-repo" + server := &config.ServerDetails{ + ArtifactoryUrl: "https://server.com/artifactory", + User: "user", + Password: "pass", + } + configFile, err := InitNewConfig(tmpDir, repoName, server, false) + assert.NoError(t, err) + f, err := os.Open(configFile.Name()) + assert.NoError(t, err) + buf := make([]byte, 1024) + n, err := f.Read(buf) + assert.NoError(t, err) + assert.Equal(t, ` + + + + + + + + + + +`, string(buf[:n])) + server.Password = "" + server.AccessToken = "abc123" + configFile, err = InitNewConfig(tmpDir, repoName, server, true) + assert.NoError(t, err) + f, err = os.Open(configFile.Name()) + assert.NoError(t, err) + buf = make([]byte, 1024) + n, err = f.Read(buf) + assert.NoError(t, err) + assert.Equal(t, ` + + + + + + + + + + +`, string(buf[:n])) +} + func TestPrepareDotnetBuildInfoModule(t *testing.T) { t.Run("generated config file", func(t *testing.T) { testPrepareDotnetBuildInfoModule(t, []string{}, true) }) t.Run("existing with configfile flag", func(t *testing.T) { diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 079838e80..c95fc0c80 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -49,7 +49,9 @@ func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, us if err != nil { return } - gradleConfigParams["usewrapper"] = useWrapper + if gradleConfigParams != nil { + gradleConfigParams["usewrapper"] = useWrapper + } } // Read config vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams) diff --git a/xray/audit/python/python_test.go b/xray/audit/python/python_test.go index de378cf07..a8f3c82de 100644 --- a/xray/audit/python/python_test.go +++ b/xray/audit/python/python_test.go @@ -60,7 +60,7 @@ func TestPipDependencyListRequirementsFallback(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "requirementsproject")) defer cleanUp() // No requirements file field specified, expect the command to use the fallback 'pip install -r requirements.txt' command - rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) + rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip, PipRequirementsFile: "requirements.txt"}) assert.NoError(t, err) assert.Len(t, rootNode, 1) if assert.True(t, len(rootNode[0].Nodes) > 2) { From 521e18ce46876981a6429f3e110e118f5a659379 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 14 Feb 2023 09:20:19 +0200 Subject: [PATCH 06/48] fix tests --- artifactory/commands/golang/testdata/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifactory/commands/golang/testdata/.gitignore b/artifactory/commands/golang/testdata/.gitignore index 2dd308744..ba7ea82da 100644 --- a/artifactory/commands/golang/testdata/.gitignore +++ b/artifactory/commands/golang/testdata/.gitignore @@ -8,5 +8,5 @@ # Test binary, build with `go test -c` *.test -# Output of the go coverage Tool, specifically when used with LiteIDE +# Output of the go coverage tool, specifically when used with LiteIDE *.out \ No newline at end of file From f3f51fe2186bfa04a87bcce5e37a19c8981780fc Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 14 Feb 2023 12:39:43 +0200 Subject: [PATCH 07/48] Fix and Add tests --- .../commands/dotnet/dotnetcommand_test.go | 23 +++++++++++++- artifactory/commands/golang/go_test.go | 12 ++++++++ artifactory/commands/python/python_test.go | 30 +++++++++++++++++++ artifactory/utils/buildinfoproperties_test.go | 15 ++++++++-- xray/audit/python/python_test.go | 5 ++++ 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 artifactory/commands/python/python_test.go diff --git a/artifactory/commands/dotnet/dotnetcommand_test.go b/artifactory/commands/dotnet/dotnetcommand_test.go index f6e04d636..3fac06174 100644 --- a/artifactory/commands/dotnet/dotnetcommand_test.go +++ b/artifactory/commands/dotnet/dotnetcommand_test.go @@ -63,7 +63,7 @@ func TestGetFlagValueExists(t *testing.T) { func TestInitNewConfig(t *testing.T) { tmpDir, err := fileutils.CreateTempDir() defer func() { - assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) + _ = fileutils.RemoveTempDir(tmpDir) }() assert.NoError(t, err) repoName := "test-repo" @@ -114,6 +114,27 @@ func TestInitNewConfig(t *testing.T) { `, string(buf[:n])) } +func TestGetSourceDetails(t *testing.T) { + server := &config.ServerDetails{ + ArtifactoryUrl: "https://server.com/artifactory", + User: "user", + Password: "pass", + } + repoName := "repo-name" + url, user, pass, err := GetSourceDetails(server, repoName, false) + assert.NoError(t, err) + assert.Equal(t, "user", user) + assert.Equal(t, "pass", pass) + assert.Equal(t, "https://server.com/artifactory/api/nuget/v3/repo-name", url) + server.Password = "" + server.AccessToken = "abc123" + url, user, pass, err = GetSourceDetails(server, repoName, true) + assert.Equal(t, "user", user) + assert.Equal(t, "abc123", pass) + assert.NoError(t, err) + assert.Equal(t, "https://server.com/artifactory/api/nuget/repo-name", url) +} + func TestPrepareDotnetBuildInfoModule(t *testing.T) { t.Run("generated config file", func(t *testing.T) { testPrepareDotnetBuildInfoModule(t, []string{}, true) }) t.Run("existing with configfile flag", func(t *testing.T) { diff --git a/artifactory/commands/golang/go_test.go b/artifactory/commands/golang/go_test.go index eb5ac364b..135b7a3e7 100644 --- a/artifactory/commands/golang/go_test.go +++ b/artifactory/commands/golang/go_test.go @@ -1,6 +1,7 @@ package golang import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/config" goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang" "github.com/jfrog/jfrog-client-go/artifactory/auth" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" @@ -72,3 +73,14 @@ func TestGetArtifactoryApiUrl(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "https://frog:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) } + +func TestGetGoRepoUrl(t *testing.T) { + server := &config.ServerDetails{ + ArtifactoryUrl: "https://server.com/artifactory", + AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA", + } + repoName := "test-repo" + repoUrl, err := GetGoRepoUrl(server, repoName) + assert.NoError(t, err) + assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@server.com/artifactoryapi/go/test-repo", repoUrl) +} diff --git a/artifactory/commands/python/python_test.go b/artifactory/commands/python/python_test.go new file mode 100644 index 000000000..ce02f53f3 --- /dev/null +++ b/artifactory/commands/python/python_test.go @@ -0,0 +1,30 @@ +package python + +import ( + "github.com/jfrog/build-info-go/utils/pythonutils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetPypiRepoUrl(t *testing.T) { + server := &config.ServerDetails{ + ArtifactoryUrl: "https://server.com/artifactory", + AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA", + } + repository := "test-repo" + url, err := GetPypiRepoUrl(server, repository) + assert.NoError(t, err) + assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@server.com/artifactoryapi/pypi/test-repo/simple", url.String()) + server.AccessToken = "" + server.User = "user" + server.Password = "password" + url, err = GetPypiRepoUrl(server, repository) + assert.NoError(t, err) + assert.Equal(t, "https://user:password@server.com/artifactoryapi/pypi/test-repo/simple", url.String()) +} + +func TestGetPypiRemoteRegistryFlag(t *testing.T) { + assert.Equal(t, "-i", GetPypiRemoteRegistryFlag(pythonutils.Pip)) + assert.Equal(t, "--pypi-mirror", GetPypiRemoteRegistryFlag(pythonutils.Pipenv)) +} diff --git a/artifactory/utils/buildinfoproperties_test.go b/artifactory/utils/buildinfoproperties_test.go index d7880ffa3..cf2968b61 100644 --- a/artifactory/utils/buildinfoproperties_test.go +++ b/artifactory/utils/buildinfoproperties_test.go @@ -73,12 +73,12 @@ func TestCreateSimplePropertiesFileWithoutProxy(t *testing.T) { } testdataPath, err := GetTestDataPath() assert.NoError(t, err) - createSimplePropertiesFile(t, filepath.Join(testdataPath, "expected_test_create_simple_properties_file_without_proxy.json"), propertiesFileConfig) + createSimplePropertiesFile(t, filepath.Join(testdataPath, "expected_test_create_simple_properties_file_without_proxy.json")) setProxy(proxyOrg, t) } -func createSimplePropertiesFile(t *testing.T, expectedPropsFilePath string, propertiesFileConfig map[string]string) { +func createSimplePropertiesFile(t *testing.T, expectedPropsFilePath string) { var yamlConfig = map[string]string{ ResolverPrefix + Url: "http://some.url.com", DeployerPrefix + Url: "http://some.other.url.com", @@ -142,3 +142,14 @@ func getOriginalProxyValue() string { func setProxy(proxy string, t *testing.T) { testsutils.SetEnvAndAssert(t, HttpProxy, proxy) } + +func TestCreateDefaultConfigWithParams(t *testing.T) { + params := map[string]any{ + "usewrapper": true, + "resolver.url": "http://localhost", + } + config := createDefaultConfigWithParams("YAML", "gradle", params) + assert.True(t, config.IsSet("usewrapper")) + assert.True(t, config.IsSet("resolver.url")) + assert.True(t, config.IsSet("type")) +} diff --git a/xray/audit/python/python_test.go b/xray/audit/python/python_test.go index a8f3c82de..4d09e9f88 100644 --- a/xray/audit/python/python_test.go +++ b/xray/audit/python/python_test.go @@ -132,3 +132,8 @@ func TestBuildPoetryDependencyList(t *testing.T) { } } } + +func TestGetPipInstallArgs(t *testing.T) { + assert.Equal(t, []string{"install", "."}, getPipInstallArgs("")) + assert.Equal(t, []string{"install", "-r", "requirements.txt"}, getPipInstallArgs("requirements.txt")) +} From 463bf9c5809b77fd95e03dcdbc2bf80d72751cc4 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 14 Feb 2023 12:40:33 +0200 Subject: [PATCH 08/48] Fix and Add tests --- artifactory/utils/buildinfoproperties_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artifactory/utils/buildinfoproperties_test.go b/artifactory/utils/buildinfoproperties_test.go index cf2968b61..0d7a9c4e7 100644 --- a/artifactory/utils/buildinfoproperties_test.go +++ b/artifactory/utils/buildinfoproperties_test.go @@ -73,12 +73,12 @@ func TestCreateSimplePropertiesFileWithoutProxy(t *testing.T) { } testdataPath, err := GetTestDataPath() assert.NoError(t, err) - createSimplePropertiesFile(t, filepath.Join(testdataPath, "expected_test_create_simple_properties_file_without_proxy.json")) + createSimplePropertiesFile(t, filepath.Join(testdataPath, "expected_test_create_simple_properties_file_without_proxy.json"), propertiesFileConfig) setProxy(proxyOrg, t) } -func createSimplePropertiesFile(t *testing.T, expectedPropsFilePath string) { +func createSimplePropertiesFile(t *testing.T, expectedPropsFilePath string, propertiesFileConfig map[string]string) { var yamlConfig = map[string]string{ ResolverPrefix + Url: "http://some.url.com", DeployerPrefix + Url: "http://some.other.url.com", From 26b9cca45eeec337015ac155f76cb577076a2823 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 14 Feb 2023 12:40:33 +0200 Subject: [PATCH 09/48] Fix and Add tests --- artifactory/utils/buildinfoproperties_test.go | 4 ++-- xray/audit/java/mvn.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/artifactory/utils/buildinfoproperties_test.go b/artifactory/utils/buildinfoproperties_test.go index cf2968b61..0d7a9c4e7 100644 --- a/artifactory/utils/buildinfoproperties_test.go +++ b/artifactory/utils/buildinfoproperties_test.go @@ -73,12 +73,12 @@ func TestCreateSimplePropertiesFileWithoutProxy(t *testing.T) { } testdataPath, err := GetTestDataPath() assert.NoError(t, err) - createSimplePropertiesFile(t, filepath.Join(testdataPath, "expected_test_create_simple_properties_file_without_proxy.json")) + createSimplePropertiesFile(t, filepath.Join(testdataPath, "expected_test_create_simple_properties_file_without_proxy.json"), propertiesFileConfig) setProxy(proxyOrg, t) } -func createSimplePropertiesFile(t *testing.T, expectedPropsFilePath string) { +func createSimplePropertiesFile(t *testing.T, expectedPropsFilePath string, propertiesFileConfig map[string]string) { var yamlConfig = map[string]string{ ResolverPrefix + Url: "http://some.url.com", DeployerPrefix + Url: "http://some.other.url.com", diff --git a/xray/audit/java/mvn.go b/xray/audit/java/mvn.go index c544feeb6..877e73aa3 100644 --- a/xray/audit/java/mvn.go +++ b/xray/audit/java/mvn.go @@ -22,7 +22,7 @@ func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool, mvnProps map[str } func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile bool, mvnProps map[string]any) (err error) { - goals := []string{"-B", "compile", "test-compile"} + goals := []string{"-B", "compile", "test-compile", "-e"} log.Debug(fmt.Sprintf("mvn command goals: %v", goals)) configFilePath := "" if !ignoreConfigFile { From 3de544a6111b9732ba4c92dfe319b919a8251f94 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Sun, 19 Feb 2023 09:02:40 +0200 Subject: [PATCH 10/48] update branch --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 845fce925..9178de053 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,6 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.3 h1:2n9BZ0YQiXGESUSR+6FLg0WWWE80u+mIz35f0uHWcIE= github.com/jedib0t/go-pretty/v6 v6.4.3/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jfrog/build-info-go v1.8.7 h1:a9aGDs5ROWcTbWP6xjmutf+yporTpEPG7lVlRnQa714= -github.com/jfrog/build-info-go v1.8.7/go.mod h1:iSTj26qEX3eUtyAGMH0qKsW4WJT+MceYxLWP9FfiAq4= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jfrog/jfrog-client-go v1.26.2 h1:UHijddbkdgmZ/zzbpFX8YJFtvaaGWya8gy/vtERjl0M= From 53cd452e5270b6d11bf4dd2089963fa8a69b5ff0 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Sun, 19 Feb 2023 09:02:40 +0200 Subject: [PATCH 11/48] update branch --- go.mod | 2 -- go.sum | 2 -- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index b3b436160..f20de9314 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,4 @@ require ( // replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.26.1-0.20230126120919-2cca98d435ec -replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230215151207-b77413be0bf7 - // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e diff --git a/go.sum b/go.sum index 845fce925..c8aa517b2 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,6 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/omerzi/build-info-go v1.3.1-0.20230215151207-b77413be0bf7 h1:kYPj9q052sbUst6y6A27iFXbFI/FxeamXAxoVQbyg0k= -github.com/omerzi/build-info-go v1.3.1-0.20230215151207-b77413be0bf7/go.mod h1:iSTj26qEX3eUtyAGMH0qKsW4WJT+MceYxLWP9FfiAq4= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= github.com/owenrumney/go-sarif/v2 v2.1.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= From add26880f0b8d3cf0a2eb5fa67bf16c0c9eb8bf4 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 20 Feb 2023 15:47:25 +0200 Subject: [PATCH 12/48] fix gradle wrapper issue --- xray/audit/java/gradle.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index c95fc0c80..569e6080f 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -49,9 +49,10 @@ func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, us if err != nil { return } - if gradleConfigParams != nil { - gradleConfigParams["usewrapper"] = useWrapper + if gradleConfigParams == nil { + gradleConfigParams = make(map[string]any) } + gradleConfigParams["usewrapper"] = useWrapper } // Read config vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams) From 5bdf58103307cbbfa9be609bb505ada885cf427f Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 20 Feb 2023 15:47:25 +0200 Subject: [PATCH 13/48] fix race condition in GetConfigList --- artifactory/utils/npm/config-list.go | 18 +++++++++--------- xray/audit/java/gradle.go | 5 +++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/artifactory/utils/npm/config-list.go b/artifactory/utils/npm/config-list.go index c8295c88a..a1516434a 100644 --- a/artifactory/utils/npm/config-list.go +++ b/artifactory/utils/npm/config-list.go @@ -9,22 +9,22 @@ import ( // This method runs "npm c ls" command and returns the current npm configuration (calculated by all flags and .npmrc files). // For more info see https://docs.npmjs.com/cli/config -func GetConfigList(npmFlags []string, executablePath string) ([]byte, error) { +func GetConfigList(npmFlags []string, executablePath string) (data []byte, err error) { pipeReader, pipeWriter := io.Pipe() - defer pipeReader.Close() + defer func(pipeReader *io.PipeReader) { + e := pipeReader.Close() + if err == nil { + err = e + } + }(pipeReader) npmFlags = append(npmFlags, "--json=false") configListCmdConfig := createConfigListCmdConfig(executablePath, npmFlags, pipeWriter) - var npmError error - go func() { - npmError = gofrogcmd.RunCmd(configListCmdConfig) - }() - - data, err := io.ReadAll(pipeReader) + npmError := gofrogcmd.RunCmd(configListCmdConfig) + data, err = io.ReadAll(pipeReader) if err != nil { return nil, errorutils.CheckError(err) } - if npmError != nil { return nil, errorutils.CheckError(npmError) } diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index c95fc0c80..569e6080f 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -49,9 +49,10 @@ func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, us if err != nil { return } - if gradleConfigParams != nil { - gradleConfigParams["usewrapper"] = useWrapper + if gradleConfigParams == nil { + gradleConfigParams = make(map[string]any) } + gradleConfigParams["usewrapper"] = useWrapper } // Read config vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams) From 537dc87f9cd8c620b0b7fdb9b06abbda55c48d1e Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Feb 2023 15:30:24 +0200 Subject: [PATCH 14/48] CR Changes --- artifactory/commands/dotnet/dotnetcommand.go | 4 +- .../commands/dotnet/dotnetcommand_test.go | 4 +- artifactory/commands/golang/go.go | 37 +----- artifactory/commands/golang/go_test.go | 42 ------ artifactory/commands/python/pip.go | 2 +- artifactory/commands/python/poetry.go | 78 +---------- artifactory/commands/python/python.go | 50 +------ utils/golang/utils.go | 37 ++++++ utils/golang/utils_test.go | 48 +++++++ utils/python/utils.go | 122 ++++++++++++++++++ .../python/utils_test.go | 7 +- xray/audit/go/golang.go | 3 +- xray/audit/java/mvn.go | 2 +- xray/audit/python/python.go | 14 +- 14 files changed, 233 insertions(+), 217 deletions(-) create mode 100644 utils/golang/utils_test.go create mode 100644 utils/python/utils.go rename artifactory/commands/python/poetry_test.go => utils/python/utils_test.go (98%) diff --git a/artifactory/commands/dotnet/dotnetcommand.go b/artifactory/commands/dotnet/dotnetcommand.go index 55e5f1a3e..5723f1ef5 100644 --- a/artifactory/commands/dotnet/dotnetcommand.go +++ b/artifactory/commands/dotnet/dotnetcommand.go @@ -265,7 +265,7 @@ func InitNewConfig(configDirPath, repoName string, server *config.ServerDetails, // Adds a source to the nuget config template func addSourceToNugetTemplate(configFile *os.File, server *config.ServerDetails, useNugetV2 bool, repoName string) error { - sourceUrl, user, password, err := GetSourceDetails(server, repoName, useNugetV2) + sourceUrl, user, password, err := getSourceDetails(server, repoName, useNugetV2) if err != nil { return err } @@ -281,7 +281,7 @@ func addSourceToNugetTemplate(configFile *os.File, server *config.ServerDetails, return err } -func GetSourceDetails(details *config.ServerDetails, repoName string, useNugetV2 bool) (sourceURL, user, password string, err error) { +func getSourceDetails(details *config.ServerDetails, repoName string, useNugetV2 bool) (sourceURL, user, password string, err error) { var u *url.URL u, err = url.Parse(details.ArtifactoryUrl) if errorutils.CheckError(err) != nil { diff --git a/artifactory/commands/dotnet/dotnetcommand_test.go b/artifactory/commands/dotnet/dotnetcommand_test.go index 3fac06174..febcb1fa3 100644 --- a/artifactory/commands/dotnet/dotnetcommand_test.go +++ b/artifactory/commands/dotnet/dotnetcommand_test.go @@ -121,14 +121,14 @@ func TestGetSourceDetails(t *testing.T) { Password: "pass", } repoName := "repo-name" - url, user, pass, err := GetSourceDetails(server, repoName, false) + url, user, pass, err := getSourceDetails(server, repoName, false) assert.NoError(t, err) assert.Equal(t, "user", user) assert.Equal(t, "pass", pass) assert.Equal(t, "https://server.com/artifactory/api/nuget/v3/repo-name", url) server.Password = "" server.AccessToken = "abc123" - url, user, pass, err = GetSourceDetails(server, repoName, true) + url, user, pass, err = getSourceDetails(server, repoName, true) assert.Equal(t, "user", user) assert.Equal(t, "abc123", pass) assert.NoError(t, err) diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index 4cc82cb41..ce984cb42 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -16,7 +16,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "net/http" - "net/url" "path" "path/filepath" "strings" @@ -154,7 +153,7 @@ func (gc *GoCommand) run() error { if err != nil { return err } - repoUrl, err := GetGoRepoUrl(resolverDetails, gc.resolverParams.TargetRepo()) + repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(resolverDetails, gc.resolverParams.TargetRepo()) if err != nil { return err } @@ -208,40 +207,6 @@ func (gc *GoCommand) run() error { return err } -func GetGoRepoUrl(serverDetails *config.ServerDetails, repo string) (string, error) { - authServerDetails, err := serverDetails.CreateArtAuthConfig() - if err != nil { - return "", err - } - return getArtifactoryApiUrl(repo, authServerDetails) -} - -// Gets the URL of the specified repository Go API in Artifactory. -// The URL contains credentials (username and access token or password). -func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails) (string, error) { - rtUrl, err := url.Parse(details.GetUrl()) - if err != nil { - return "", errorutils.CheckError(err) - } - - username := details.GetUser() - password := details.GetPassword() - - // Get credentials from access-token if exists. - if details.GetAccessToken() != "" { - log.Debug("Using proxy with access-token.") - if username == "" { - username = auth.ExtractUsernameFromAccessToken(details.GetAccessToken()) - } - password = details.GetAccessToken() - } - if password != "" { - rtUrl.User = url.UserPassword(username, password) - } - rtUrl.Path += "api/go/" + repoName - return rtUrl.String(), nil -} - // copyGoPackageFiles copies the package files from the go mod cache directory to the given destPath. // The path to those cache files is retrieved using the supplied package name and Artifactory details. func copyGoPackageFiles(destPath, packageName, rtTargetRepo string, authArtDetails auth.ServiceDetails) error { diff --git a/artifactory/commands/golang/go_test.go b/artifactory/commands/golang/go_test.go index 135b7a3e7..974808889 100644 --- a/artifactory/commands/golang/go_test.go +++ b/artifactory/commands/golang/go_test.go @@ -1,9 +1,7 @@ package golang import ( - "github.com/jfrog/jfrog-cli-core/v2/utils/config" goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang" - "github.com/jfrog/jfrog-client-go/artifactory/auth" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" "os" @@ -44,43 +42,3 @@ func TestGetPackageFilesPath(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expectedPackagePath, actualPackagePath) } - -func TestGetArtifactoryApiUrl(t *testing.T) { - details := auth.NewArtifactoryDetails() - details.SetUrl("https://test.com/artifactory/") - - // Test username and password - details.SetUser("frog") - details.SetPassword("passfrog") - url, err := getArtifactoryApiUrl("test-repo", details) - assert.NoError(t, err) - assert.Equal(t, "https://frog:passfrog@test.com/artifactory/api/go/test-repo", url) - - // Test access token - // Set fake access token with username "test" - details.SetUser("") - details.SetAccessToken("eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA") - url, err = getArtifactoryApiUrl("test-repo", details) - assert.NoError(t, err) - assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) - - // Test access token with username - // Set fake access token with username "test" - // Expect username to be "frog" - details.SetUser("frog") - details.SetAccessToken("eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA") - url, err = getArtifactoryApiUrl("test-repo", details) - assert.NoError(t, err) - assert.Equal(t, "https://frog:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) -} - -func TestGetGoRepoUrl(t *testing.T) { - server := &config.ServerDetails{ - ArtifactoryUrl: "https://server.com/artifactory", - AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA", - } - repoName := "test-repo" - repoUrl, err := GetGoRepoUrl(server, repoName) - assert.NoError(t, err) - assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@server.com/artifactoryapi/go/test-repo", repoUrl) -} diff --git a/artifactory/commands/python/pip.go b/artifactory/commands/python/pip.go index 3b5e91e70..8b7b2267a 100644 --- a/artifactory/commands/python/pip.go +++ b/artifactory/commands/python/pip.go @@ -16,7 +16,7 @@ type PipCommand struct { } func NewPipCommand() *PipCommand { - return &PipCommand{PythonCommand: PythonCommand{pythonTool: pythonutils.Pip}} + return &PipCommand{PythonCommand: *NewPythonCommand(pythonutils.Pip)} } func (pc *PipCommand) Run() (err error) { diff --git a/artifactory/commands/python/poetry.go b/artifactory/commands/python/poetry.go index ad3ae9870..d38dbca12 100644 --- a/artifactory/commands/python/poetry.go +++ b/artifactory/commands/python/poetry.go @@ -5,20 +5,16 @@ import ( "fmt" "github.com/jfrog/build-info-go/build" "github.com/jfrog/build-info-go/entities" - buildinfoutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/build-info-go/utils/pythonutils" gofrogcmd "github.com/jfrog/gofrog/io" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python/dependencies" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" + python "github.com/jfrog/jfrog-cli-core/v2/utils/python" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" - "github.com/spf13/viper" "io" - "os" "os/exec" - "path/filepath" - "strings" ) type PoetryCommand struct { @@ -27,16 +23,11 @@ type PoetryCommand struct { poetryConfigRepoName string } -const ( - baseConfigRepoName = "jfrog-server" - poetryConfigAuthPrefix = "http-basic." - poetryConfigRepoPrefix = "repositories." - pyproject = "pyproject.toml" -) +const baseConfigRepoName = "jfrog-server" func NewPoetryCommand() *PoetryCommand { return &PoetryCommand{ - PythonCommand: PythonCommand{pythonTool: pythonutils.Poetry}, + PythonCommand: *NewPythonCommand(pythonutils.Poetry), poetryConfigRepoName: baseConfigRepoName, } } @@ -136,12 +127,12 @@ func (pc *PoetryCommand) SetCommandName(commandName string) *PoetryCommand { } func (pc *PoetryCommand) SetPypiRepoUrlWithCredentials() error { - rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(pc.serverDetails, pc.repository) + rtUrl, username, password, err := python.GetPypiRepoUrlWithCredentials(pc.serverDetails, pc.repository) if err != nil { return err } if password != "" { - return ConfigPoetryRepo( + return python.ConfigPoetryRepo( rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, @@ -150,49 +141,6 @@ func (pc *PoetryCommand) SetPypiRepoUrlWithCredentials() error { return nil } -func ConfigPoetryRepo(url, username, password, configRepoName string) error { - // Add the poetry repository config - err := runPoetryConfigCommand([]string{poetryConfigRepoPrefix + configRepoName, url}, false) - if err != nil { - return err - } - - // Set the poetry repository credentials - err = runPoetryConfigCommand([]string{poetryConfigAuthPrefix + configRepoName, username, password}, true) - if err != nil { - return err - } - - // Add the repository config to the pyproject.toml - currentDir, err := os.Getwd() - if err != nil { - return errorutils.CheckError(err) - } - return addRepoToPyprojectFile(filepath.Join(currentDir, pyproject), configRepoName, url) -} - -func addRepoToPyprojectFile(filepath, poetryRepoName, repoUrl string) error { - viper.SetConfigType("toml") - viper.SetConfigFile(filepath) - err := viper.ReadInConfig() - if err != nil { - return errorutils.CheckErrorf("Failed to read pyproject.toml: %s", err.Error()) - } - viper.Set("tool.poetry.source", []map[string]string{{"name": poetryRepoName, "url": repoUrl}}) - err = viper.WriteConfig() - if err != nil { - return errorutils.CheckErrorf("Failed to add tool.poetry.source to pyproject.toml: %s", err.Error()) - } - log.Info(fmt.Sprintf("Added tool.poetry.source name:%q url:%q", poetryRepoName, repoUrl)) - log.Info("Running Poetry update") - cmd := buildinfoutils.NewCommand("poetry", "update", []string{}) - err = gofrogcmd.RunCmd(cmd) - if err != nil { - return errorutils.CheckErrorf("Poetry config command failed with: %s", err.Error()) - } - return err -} - func (pc *PoetryCommand) CommandName() string { return "rt_python_poetry" } @@ -225,19 +173,3 @@ func (pc *PoetryCommand) GetStdWriter() io.WriteCloser { func (pc *PoetryCommand) GetErrWriter() io.WriteCloser { return nil } - -func runPoetryConfigCommand(args []string, maskArgs bool) error { - logMessage := "config " - if maskArgs { - logMessage += "***" - } else { - logMessage += strings.Join(args, " ") - } - log.Info(fmt.Sprintf("Running Poetry %s", logMessage)) - cmd := buildinfoutils.NewCommand("poetry", "config", args) - err := gofrogcmd.RunCmd(cmd) - if err != nil { - return errorutils.CheckErrorf("Poetry config command failed with: %s", err.Error()) - } - return nil -} diff --git a/artifactory/commands/python/python.go b/artifactory/commands/python/python.go index 17c0e8911..8d1afeea6 100644 --- a/artifactory/commands/python/python.go +++ b/artifactory/commands/python/python.go @@ -2,8 +2,8 @@ package python import ( "errors" + python "github.com/jfrog/jfrog-cli-core/v2/utils/python" "io" - "net/url" "os/exec" "github.com/jfrog/build-info-go/build" @@ -13,16 +13,10 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python/dependencies" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" ) -const ( - PipenvRemoteRegistryFlag = "--pypi-mirror" - PipRemoteRegistryFlag = "-i" -) - type PythonCommand struct { serverDetails *config.ServerDetails pythonTool pythonutils.PythonTool @@ -108,11 +102,11 @@ func (pc *PythonCommand) SetCommandName(commandName string) *PythonCommand { } func (pc *PythonCommand) SetPypiRepoUrlWithCredentials() error { - rtUrl, err := GetPypiRepoUrl(pc.serverDetails, pc.repository) + rtUrl, err := python.GetPypiRepoUrl(pc.serverDetails, pc.repository) if err != nil { return err } - pc.args = append(pc.args, GetPypiRemoteRegistryFlag(pc.pythonTool), rtUrl.String()) + pc.args = append(pc.args, python.GetPypiRemoteRegistryFlag(pc.pythonTool), rtUrl.String()) return nil } @@ -144,41 +138,3 @@ func (pc *PythonCommand) GetStdWriter() io.WriteCloser { func (pc *PythonCommand) GetErrWriter() io.WriteCloser { return nil } - -func GetPypiRepoUrl(serverDetails *config.ServerDetails, repository string) (*url.URL, error) { - rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(serverDetails, repository) - if err != nil { - return nil, err - } - if password != "" { - rtUrl.User = url.UserPassword(username, password) - } - return rtUrl, err -} - -func GetPypiRepoUrlWithCredentials(serverDetails *config.ServerDetails, repository string) (*url.URL, string, string, error) { - rtUrl, err := url.Parse(serverDetails.GetArtifactoryUrl()) - if err != nil { - return nil, "", "", errorutils.CheckError(err) - } - - username := serverDetails.GetUser() - password := serverDetails.GetPassword() - - // Get credentials from access-token if exists. - if serverDetails.GetAccessToken() != "" { - if username == "" { - username = auth.ExtractUsernameFromAccessToken(serverDetails.GetAccessToken()) - } - password = serverDetails.GetAccessToken() - } - rtUrl.Path += "api/pypi/" + repository + "/simple" - return rtUrl, username, password, err -} - -func GetPypiRemoteRegistryFlag(tool pythonutils.PythonTool) string { - if tool == pythonutils.Pip { - return PipRemoteRegistryFlag - } - return PipenvRemoteRegistryFlag -} diff --git a/utils/golang/utils.go b/utils/golang/utils.go index 61ec51232..ab4176111 100644 --- a/utils/golang/utils.go +++ b/utils/golang/utils.go @@ -2,9 +2,12 @@ package goutils import ( "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" "io" + "net/url" "os/exec" ) @@ -95,3 +98,37 @@ func GetDependenciesGraph(projectDir string) (map[string][]string, error) { } return deps, nil } + +func GetArtifactoryRemoteRepoUrl(serverDetails *config.ServerDetails, repo string) (string, error) { + authServerDetails, err := serverDetails.CreateArtAuthConfig() + if err != nil { + return "", err + } + return getArtifactoryApiUrl(repo, authServerDetails) +} + +// Gets the URL of the specified repository Go API in Artifactory. +// The URL contains credentials (username and access token or password). +func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails) (string, error) { + rtUrl, err := url.Parse(details.GetUrl()) + if err != nil { + return "", errorutils.CheckError(err) + } + + username := details.GetUser() + password := details.GetPassword() + + // Get credentials from access-token if exists. + if details.GetAccessToken() != "" { + log.Debug("Using proxy with access-token.") + if username == "" { + username = auth.ExtractUsernameFromAccessToken(details.GetAccessToken()) + } + password = details.GetAccessToken() + } + if password != "" { + rtUrl.User = url.UserPassword(username, password) + } + rtUrl.Path += "api/go/" + repoName + return rtUrl.String(), nil +} diff --git a/utils/golang/utils_test.go b/utils/golang/utils_test.go new file mode 100644 index 000000000..bda3953a8 --- /dev/null +++ b/utils/golang/utils_test.go @@ -0,0 +1,48 @@ +package goutils + +import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/artifactory/auth" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetArtifactoryRemoteRepoUrl(t *testing.T) { + server := &config.ServerDetails{ + ArtifactoryUrl: "https://server.com/artifactory", + AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA", + } + repoName := "test-repo" + repoUrl, err := GetArtifactoryRemoteRepoUrl(server, repoName) + assert.NoError(t, err) + assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@server.com/artifactoryapi/go/test-repo", repoUrl) +} + +func TestGetArtifactoryApiUrl(t *testing.T) { + details := auth.NewArtifactoryDetails() + details.SetUrl("https://test.com/artifactory/") + + // Test username and password + details.SetUser("frog") + details.SetPassword("passfrog") + url, err := getArtifactoryApiUrl("test-repo", details) + assert.NoError(t, err) + assert.Equal(t, "https://frog:passfrog@test.com/artifactory/api/go/test-repo", url) + + // Test access token + // Set fake access token with username "test" + details.SetUser("") + details.SetAccessToken("eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA") + url, err = getArtifactoryApiUrl("test-repo", details) + assert.NoError(t, err) + assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) + + // Test access token with username + // Set fake access token with username "test" + // Expect username to be "frog" + details.SetUser("frog") + details.SetAccessToken("eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA") + url, err = getArtifactoryApiUrl("test-repo", details) + assert.NoError(t, err) + assert.Equal(t, "https://frog:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) +} diff --git a/utils/python/utils.go b/utils/python/utils.go new file mode 100644 index 000000000..d5057b894 --- /dev/null +++ b/utils/python/utils.go @@ -0,0 +1,122 @@ +package utils + +import ( + "fmt" + buildinfoutils "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/build-info-go/utils/pythonutils" + gofrogcmd "github.com/jfrog/gofrog/io" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/spf13/viper" + "net/url" + "os" + "path/filepath" + "strings" +) + +const ( + pipenvRemoteRegistryFlag = "--pypi-mirror" + pipRemoteRegistryFlag = "-i" + poetryConfigAuthPrefix = "http-basic." + poetryConfigRepoPrefix = "repositories." + pyproject = "pyproject.toml" +) + +func GetPypiRepoUrlWithCredentials(serverDetails *config.ServerDetails, repository string) (*url.URL, string, string, error) { + rtUrl, err := url.Parse(serverDetails.GetArtifactoryUrl()) + if err != nil { + return nil, "", "", errorutils.CheckError(err) + } + + username := serverDetails.GetUser() + password := serverDetails.GetPassword() + + // Get credentials from access-token if exists. + if serverDetails.GetAccessToken() != "" { + if username == "" { + username = auth.ExtractUsernameFromAccessToken(serverDetails.GetAccessToken()) + } + password = serverDetails.GetAccessToken() + } + rtUrl.Path += "api/pypi/" + repository + "/simple" + return rtUrl, username, password, err +} + +func GetPypiRemoteRegistryFlag(tool pythonutils.PythonTool) string { + if tool == pythonutils.Pip { + return pipRemoteRegistryFlag + } + return pipenvRemoteRegistryFlag +} + +func GetPypiRepoUrl(serverDetails *config.ServerDetails, repository string) (*url.URL, error) { + rtUrl, username, password, err := GetPypiRepoUrlWithCredentials(serverDetails, repository) + if err != nil { + return nil, err + } + if password != "" { + rtUrl.User = url.UserPassword(username, password) + } + return rtUrl, err +} + +func ConfigPoetryRepo(url, username, password, configRepoName string) error { + // Add the poetry repository config + err := runPoetryConfigCommand([]string{poetryConfigRepoPrefix + configRepoName, url}, false) + if err != nil { + return err + } + + // Set the poetry repository credentials + err = runPoetryConfigCommand([]string{poetryConfigAuthPrefix + configRepoName, username, password}, true) + if err != nil { + return err + } + + // Add the repository config to the pyproject.toml + currentDir, err := os.Getwd() + if err != nil { + return errorutils.CheckError(err) + } + return addRepoToPyprojectFile(filepath.Join(currentDir, pyproject), configRepoName, url) +} + +func runPoetryConfigCommand(args []string, maskArgs bool) error { + logMessage := "config " + if maskArgs { + logMessage += "***" + } else { + logMessage += strings.Join(args, " ") + } + log.Info(fmt.Sprintf("Running Poetry %s", logMessage)) + cmd := buildinfoutils.NewCommand("poetry", "config", args) + err := gofrogcmd.RunCmd(cmd) + if err != nil { + return errorutils.CheckErrorf("Poetry config command failed with: %s", err.Error()) + } + return nil +} + +func addRepoToPyprojectFile(filepath, poetryRepoName, repoUrl string) error { + viper.SetConfigType("toml") + viper.SetConfigFile(filepath) + err := viper.ReadInConfig() + if err != nil { + return errorutils.CheckErrorf("Failed to read pyproject.toml: %s", err.Error()) + } + viper.Set("tool.poetry.source", []map[string]string{{"name": poetryRepoName, "url": repoUrl}}) + err = viper.WriteConfig() + if err != nil { + return errorutils.CheckErrorf("Failed to add tool.poetry.source to pyproject.toml: %s", err.Error()) + } + log.Info(fmt.Sprintf("Added tool.poetry.source name:%q url:%q", poetryRepoName, repoUrl)) + log.Info("Running Poetry update") + cmd := buildinfoutils.NewCommand("poetry", "update", []string{}) + err = gofrogcmd.RunCmd(cmd) + if err != nil { + return errorutils.CheckErrorf("Poetry config command failed with: %s", err.Error()) + } + return err +} diff --git a/artifactory/commands/python/poetry_test.go b/utils/python/utils_test.go similarity index 98% rename from artifactory/commands/python/poetry_test.go rename to utils/python/utils_test.go index 3fbc57703..387545da7 100644 --- a/artifactory/commands/python/poetry_test.go +++ b/utils/python/utils_test.go @@ -1,12 +1,11 @@ -package python +package utils import ( - "path/filepath" - "testing" - "github.com/jfrog/jfrog-cli-core/v2/xray/audit" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/stretchr/testify/assert" + "path/filepath" + "testing" ) func TestAddRepoToPyprojectFile(t *testing.T) { diff --git a/xray/audit/go/golang.go b/xray/audit/go/golang.go index a760fa78e..b2f022b67 100644 --- a/xray/audit/go/golang.go +++ b/xray/audit/go/golang.go @@ -2,7 +2,6 @@ package _go import ( "github.com/jfrog/build-info-go/utils" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/golang" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "os" "strings" @@ -60,7 +59,7 @@ func BuildDependencyTree(server *config.ServerDetails, remoteGoRepo string) (dep } func setGoProxy(server *config.ServerDetails, remoteGoRepo string) error { - repoUrl, err := golang.GetGoRepoUrl(server, remoteGoRepo) + repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(server, remoteGoRepo) if err != nil { return err } diff --git a/xray/audit/java/mvn.go b/xray/audit/java/mvn.go index 877e73aa3..c544feeb6 100644 --- a/xray/audit/java/mvn.go +++ b/xray/audit/java/mvn.go @@ -22,7 +22,7 @@ func BuildMvnDependencyTree(insecureTls, ignoreConfigFile bool, mvnProps map[str } func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile bool, mvnProps map[string]any) (err error) { - goals := []string{"-B", "compile", "test-compile", "-e"} + goals := []string{"-B", "compile", "test-compile"} log.Debug(fmt.Sprintf("mvn command goals: %v", goals)) configFilePath := "" if !ignoreConfigFile { diff --git a/xray/audit/python/python.go b/xray/audit/python/python.go index 35a44f57d..3da99420d 100644 --- a/xray/audit/python/python.go +++ b/xray/audit/python/python.go @@ -3,8 +3,8 @@ package python import ( "fmt" "github.com/jfrog/build-info-go/utils/pythonutils" - rtpython "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/python" "github.com/jfrog/jfrog-cli-core/v2/utils/config" + utils "github.com/jfrog/jfrog-cli-core/v2/utils/python" "github.com/jfrog/jfrog-cli-core/v2/xray/audit" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" @@ -127,12 +127,12 @@ func installPoetryDeps(auditPython *AuditPython) (restoreEnv func() error, err e return nil } if auditPython.RemotePypiRepo != "" { - rtUrl, username, password, err := rtpython.GetPypiRepoUrlWithCredentials(auditPython.Server, auditPython.RemotePypiRepo) + rtUrl, username, password, err := utils.GetPypiRepoUrlWithCredentials(auditPython.Server, auditPython.RemotePypiRepo) if err != nil { return restoreEnv, err } if password != "" { - err = rtpython.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, auditPython.RemotePypiRepo) + err = utils.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, auditPython.RemotePypiRepo) if err != nil { return restoreEnv, err } @@ -218,21 +218,21 @@ func getPipExec() string { } func runPipInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName, pipRequirementsFile string) (err error) { - rtUrl, err := rtpython.GetPypiRepoUrl(server, depsRepoName) + rtUrl, err := utils.GetPypiRepoUrl(server, depsRepoName) if err != nil { return err } args := getPipInstallArgs(pipRequirementsFile) - args = append(args, rtpython.GetPypiRemoteRegistryFlag(pythonutils.Pip), rtUrl.String()) + args = append(args, utils.GetPypiRemoteRegistryFlag(pythonutils.Pip), rtUrl.String()) return runPipInstall(args...) } func runPipenvInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName string) (err error) { - rtUrl, err := rtpython.GetPypiRepoUrl(server, depsRepoName) + rtUrl, err := utils.GetPypiRepoUrl(server, depsRepoName) if err != nil { return err } - args := []string{"install", "-d", rtpython.GetPypiRemoteRegistryFlag(pythonutils.Pipenv), rtUrl.String()} + args := []string{"install", "-d", utils.GetPypiRemoteRegistryFlag(pythonutils.Pipenv), rtUrl.String()} return executeCommand("pipenv", args...) } From d683efccad3675d043f0989ec6a991a0b312c561 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Feb 2023 15:51:46 +0200 Subject: [PATCH 15/48] CR Changes --- artifactory/commands/npm/common.go | 4 +-- artifactory/commands/python/python_test.go | 30 ---------------------- 2 files changed, 2 insertions(+), 32 deletions(-) delete mode 100644 artifactory/commands/python/python_test.go diff --git a/artifactory/commands/npm/common.go b/artifactory/commands/npm/common.go index f8325ae0b..d9b6d1483 100644 --- a/artifactory/commands/npm/common.go +++ b/artifactory/commands/npm/common.go @@ -202,6 +202,6 @@ func (ca *CommonArgs) setRestoreNpmrcFunc() error { return err } -func (com *CommonArgs) GetRestoreNpmrcFunc() func() error { - return com.restoreNpmrcFunc +func (ca *CommonArgs) GetRestoreNpmrcFunc() func() error { + return ca.restoreNpmrcFunc } diff --git a/artifactory/commands/python/python_test.go b/artifactory/commands/python/python_test.go deleted file mode 100644 index ce02f53f3..000000000 --- a/artifactory/commands/python/python_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package python - -import ( - "github.com/jfrog/build-info-go/utils/pythonutils" - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestGetPypiRepoUrl(t *testing.T) { - server := &config.ServerDetails{ - ArtifactoryUrl: "https://server.com/artifactory", - AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA", - } - repository := "test-repo" - url, err := GetPypiRepoUrl(server, repository) - assert.NoError(t, err) - assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@server.com/artifactoryapi/pypi/test-repo/simple", url.String()) - server.AccessToken = "" - server.User = "user" - server.Password = "password" - url, err = GetPypiRepoUrl(server, repository) - assert.NoError(t, err) - assert.Equal(t, "https://user:password@server.com/artifactoryapi/pypi/test-repo/simple", url.String()) -} - -func TestGetPypiRemoteRegistryFlag(t *testing.T) { - assert.Equal(t, "-i", GetPypiRemoteRegistryFlag(pythonutils.Pip)) - assert.Equal(t, "--pypi-mirror", GetPypiRemoteRegistryFlag(pythonutils.Pipenv)) -} From d469e3f640f5f591cbee71efc2de76bef66bea5d Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Feb 2023 16:17:48 +0200 Subject: [PATCH 16/48] fix tests --- utils/python/utils_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/python/utils_test.go b/utils/python/utils_test.go index 387545da7..6e188d305 100644 --- a/utils/python/utils_test.go +++ b/utils/python/utils_test.go @@ -25,7 +25,7 @@ func TestAddRepoToPyprojectFile(t *testing.T) { func initPoetryTest(t *testing.T) (string, func()) { // Create and change directory to test workspace - testAbs, err := filepath.Abs(filepath.Join("..", "..", "..", "xray", "commands", "testdata", "poetry-project")) + testAbs, err := filepath.Abs(filepath.Join("..", "..", "xray", "commands", "testdata", "poetry-project")) assert.NoError(t, err) poetryProjectPath, cleanUp := audit.CreateTestWorkspace(t, "poetry-project") assert.NoError(t, fileutils.CopyDir(testAbs, poetryProjectPath, true, nil)) From 8bd583b9860be5d97806811b68a243242014565a Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Thu, 23 Mar 2023 15:36:30 +0200 Subject: [PATCH 17/48] Use gradle-dep-tree in Audit --- artifactory/commands/gradle/gradle.go | 2 +- artifactory/utils/buildinfoproperties.go | 9 - go.mod | 2 +- go.sum | 4 +- utils/gradle/utils.go | 2 +- xray/audit/java/gradle.go | 252 +++++++++++++++--- xray/audit/java/gradle_test.go | 181 +++++++++++-- xray/audit/java/javautils.go | 18 +- xray/commands/audit/generic/auditmanager.go | 55 +--- .../gradle-example-ci-server/build.gradle | 4 - .../build/gradle-dep-tree/YXBp | 33 +++ .../Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy | 5 + .../build/gradle-dep-tree/c2VydmljZXM= | 5 + .../build/gradle-dep-tree/c2hhcmVk | 5 + .../build/gradle-dep-tree/d2Vic2VydmljZQ== | 53 ++++ 15 files changed, 511 insertions(+), 119 deletions(-) create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== diff --git a/artifactory/commands/gradle/gradle.go b/artifactory/commands/gradle/gradle.go index ba8eb7a6e..2964854c2 100644 --- a/artifactory/commands/gradle/gradle.go +++ b/artifactory/commands/gradle/gradle.go @@ -56,7 +56,7 @@ func (gc *GradleCommand) SetServerDetails(serverDetails *config.ServerDetails) * func (gc *GradleCommand) init() (vConfig *viper.Viper, err error) { // Read config - vConfig, err = utils.ReadGradleConfig(gc.configPath, nil) + vConfig, err = utils.ReadConfigFile(gc.configPath, utils.YAML) if err != nil { return } diff --git a/artifactory/utils/buildinfoproperties.go b/artifactory/utils/buildinfoproperties.go index 877588f54..7bfa73df2 100644 --- a/artifactory/utils/buildinfoproperties.go +++ b/artifactory/utils/buildinfoproperties.go @@ -155,15 +155,6 @@ func ReadConfigFile(configPath string, configType ConfigType) (config *viper.Vip return config, errorutils.CheckError(err) } -func ReadGradleConfig(path string, gradleConfigParams map[string]any) (config *viper.Viper, err error) { - if path == "" { - config = createDefaultConfigWithParams(YAML, Gradle.String(), gradleConfigParams) - } else { - config, err = ReadConfigFile(path, YAML) - } - return -} - func ReadMavenConfig(path string, mvnProps map[string]any) (config *viper.Viper, err error) { if path == "" { config = createDefaultConfigWithParams(YAML, Maven.String(), mvnProps) diff --git a/go.mod b/go.mod index 0289e4413..43687b943 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,6 @@ require ( // replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.26.1-0.20230126120919-2cca98d435ec -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7 +replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e diff --git a/go.sum b/go.sum index 639e80ce2..62163aca8 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,6 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jfrog/build-info-go v1.8.10 h1:0kvKqs8n98NfixoEciorBwZF9vuYs23060Bc4Wl5xMM= -github.com/jfrog/build-info-go v1.8.10/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jfrog/jfrog-client-go v1.27.0 h1:febOOMZ7yisfMjEXCryOvcK+tMzDz591L08Jn0OeMt8= @@ -255,6 +253,8 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb h1:XWo+nYnmGE4Dy0K85oENHJcogIHpMqB5TtBkqWZy9O4= +github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= github.com/owenrumney/go-sarif/v2 v2.1.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= diff --git a/utils/gradle/utils.go b/utils/gradle/utils.go index a1a217708..b61fc69a5 100644 --- a/utils/gradle/utils.go +++ b/utils/gradle/utils.go @@ -39,7 +39,7 @@ func RunGradle(vConfig *viper.Viper, tasks, deployableArtifactsFile string, conf if err != nil { return err } - dependencyLocalPath, err := GetGradleDependencyLocalPath() + dependencyLocalPath, err := getGradleDependencyLocalPath() if err != nil { return err } diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 203a375af..32ab30f54 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -1,65 +1,251 @@ package java import ( + "encoding/json" "fmt" + "github.com/jfrog/build-info-go/build" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - gradleutils "github.com/jfrog/jfrog-cli-core/v2/utils/gradle" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" + "os" + "os/exec" + "path/filepath" + "strings" ) -const gradlew = "gradlew" +const ( + remoteDepTreePath = "artifactory/oss-releases" + gradlew = "gradlew" + depTreeInitFile = "gradledeptree.init" + depTreeOutputFile = "gradledeptree.out" + depTreeInitScript = `initscript { + repositories { %s + mavenCentral() + } + dependencies { + classpath 'com.jfrog:gradle-dep-tree:+' + } +} -func buildGradleDependencyTree(excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (dependencyTree []*services.GraphNode, err error) { - buildConfiguration, cleanBuild := createBuildConfiguration("audit-gradle") - defer cleanBuild(err) +allprojects { + repositories { %s + } + apply plugin: com.jfrog.GradleDepTree +}` + artifactoryRepository = ` + maven { + url "%s/%s" + credentials { + username = '%s' + password = '%s' + } + }` +) - err = runGradle(buildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile, gradleConfigParams) - if err != nil { - return +type gradleDepsMap map[string]any + +func (gdg *gradleDepsMap) appendTree(jsonDepTree []byte) error { + var rootNode map[string]any + if err := json.Unmarshal(jsonDepTree, &rootNode); err != nil { + return err } - dependencyTree, err = createGavDependencyTree(buildConfiguration) - return + for gav, node := range rootNode["children"].(map[string]any) { + if _, exists := (*gdg)[gav]; !exists { + (*gdg)[gav] = node + } + } + return nil } -func runGradle(buildConfiguration *utils.BuildConfiguration, excludeTestDeps, useWrapper, ignoreConfigFile bool, gradleConfigParams map[string]any) (err error) { - tasks := "clean compileJava " - if !excludeTestDeps { - tasks += "compileTestJava " - } - tasks += "artifactoryPublish" - log.Debug(fmt.Sprintf("gradle command tasks: %v", tasks)) - configFilePath := "" - if !ignoreConfigFile { - var exists bool - configFilePath, exists, err = utils.GetProjectConfFilePath(utils.Gradle) +// The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. +// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in gradleDepsMap struct. +func (gdg *gradleDepsMap) parseDepTreeFiles(jsonFiles []byte) error { + outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") + for _, path := range outputFiles { + tree, err := os.ReadFile(strings.TrimSpace(path)) + if err != nil { + return err + } + if err = gdg.appendTree(tree); err != nil { + return err + } + + } + return nil +} + +type depTreeManager struct { + server *config.ServerDetails + releasesRepo string + depsRepo string + useWrapper bool +} + +func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, depsRepo, releasesRepo string) (dependencyTree []*services.GraphNode, err error) { + if (server != nil && server.IsEmpty()) || depsRepo == "" { + depsRepo, server, err = getGradleConfig() if err != nil { return } - if exists { - log.Debug("Using resolver config from", configFilePath) + } + + manager := &depTreeManager{ + server: server, + releasesRepo: releasesRepo, + depsRepo: depsRepo, + useWrapper: useWrapper, + } + + outputFileContent, err := manager.runGradleDepTree() + if err != nil { + return nil, err + } + return manager.getGraphFromDepTree(outputFileContent) +} + +func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err error) { + // Create the script file in the repository + if err = dtp.createDepTreeScript(); err != nil { + return + } + defer func() { + e := os.Remove(depTreeInitFile) + if err == nil { + err = e + } + }() + + if dtp.useWrapper { + dtp.useWrapper, err = isGradleWrapperExist() + if err != nil { + return } } - // Check whether gradle wrapper exists - if useWrapper { - useWrapper, err = isGradleWrapperExist() + + return dtp.execGradleDepTree() +} + +func (dtp *depTreeManager) createDepTreeScript() (err error) { + depsRepo := "" + releasesRepo := "" + if dtp.server != nil { + releasesRepo, err = getDepTreeArtifactoryRepository(fmt.Sprintf("%s/%s", dtp.releasesRepo, remoteDepTreePath), dtp.server) + if err != nil { + return + } + depsRepo, err = getDepTreeArtifactoryRepository(dtp.depsRepo, dtp.server) if err != nil { return } - if gradleConfigParams == nil { - gradleConfigParams = make(map[string]any) + } + depTreeInitScript := fmt.Sprintf(depTreeInitScript, releasesRepo, depsRepo) + return os.WriteFile(depTreeInitFile, []byte(depTreeInitScript), 0666) +} + +func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err error) { + gradleExecPath, err := build.GetGradleExecPath(dtp.useWrapper) + if err != nil { + return + } + if err = os.Chmod(gradleExecPath, 0777); err != nil { + return + } + + outputFileAbsolutePath, err := filepath.Abs(depTreeOutputFile) + if err != nil { + return nil, err + } + tasks := []string{"clean", "generateDepTrees", "-I", depTreeInitFile, "-q", fmt.Sprintf("-Dcom.jfrog.depsTreeOutputFile=%s", outputFileAbsolutePath), "-Dcom.jfrog.includeAllBuildFiles=true"} + log.Info("Running gradle dep tree command: ", gradleExecPath, tasks) + if output, err := exec.Command(gradleExecPath, tasks...).CombinedOutput(); err != nil { + return nil, fmt.Errorf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output)) + } + defer func() { + e := os.Remove(outputFileAbsolutePath) + if err == nil { + err = e + } + }() + + return os.ReadFile(depTreeOutputFile) +} + +// Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input +func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*services.GraphNode, error) { + dependencyMap := gradleDepsMap{} + if err := dependencyMap.parseDepTreeFiles(outputFileContent); err != nil { + return nil, err + } + + var depsGraph []*services.GraphNode + for dependency, dependencyDetails := range dependencyMap { + directDependency := &services.GraphNode{ + Id: GavPackageTypeIdentifier + dependency, + Nodes: []*services.GraphNode{}, } - gradleConfigParams["usewrapper"] = useWrapper + populateGradleDependencyTree(directDependency, dependencyDetails.(map[string]any)["children"].(map[string]any)) + depsGraph = append(depsGraph, directDependency) } - // Read config - vConfig, err := utils.ReadGradleConfig(configFilePath, gradleConfigParams) + return depsGraph, nil +} + +func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren map[string]any) { + if currNode.NodeHasLoop() { + return + } + + for gav, details := range currNodeChildren { + childNode := &services.GraphNode{ + Id: GavPackageTypeIdentifier + gav, + Nodes: []*services.GraphNode{}, + Parent: currNode, + } + childNodeChildren := details.(map[string]any)["children"].(map[string]any) + populateGradleDependencyTree(childNode, childNodeChildren) + currNode.Nodes = append(currNode.Nodes, childNode) + } +} + +func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) { + pass := server.Password + user := server.User + if server.AccessToken != "" { + pass = server.AccessToken + } + if pass == "" && user == "" { + return "", fmt.Errorf("either username/password or access token must be set for %s", server.Url) + } + return fmt.Sprintf(artifactoryRepository, + server.ArtifactoryUrl, + remoteRepo, + user, + pass), nil +} + +func getGradleConfig() (string, *config.ServerDetails, error) { + var exists bool + configFilePath, exists, err := utils.GetProjectConfFilePath(utils.Gradle) if err != nil { - return err + return "", nil, err + } + if !exists { + return "", nil, nil + } + log.Debug("Using resolver config from", configFilePath) + configContent, err := utils.ReadConfigFile(configFilePath, utils.YAML) + if err != nil { + return "", nil, err + } + var repository string + if configContent.IsSet("resolver.repo") { + repository = configContent.Get("resolver.repo").(string) } - return gradleutils.RunGradle(vConfig, tasks, "", buildConfiguration, 0, true) + server, err := utils.GetServerDetails(configContent) + return repository, server, err } // This function assumes that the Gradle wrapper is in the root directory. diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 61901c547..135763e7d 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -1,6 +1,10 @@ package java import ( + "fmt" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/jfrog/jfrog-client-go/xray/services" "os" "path/filepath" "testing" @@ -17,15 +21,15 @@ func TestGradleTreesWithoutConfig(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildGradleDependencyTree(false, true, true, nil) + modulesDependencyTrees, err := buildGradleDependencyTree(false, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 5) + assert.Len(t, modulesDependencyTrees, 7) // Check module - module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.example.gradle:webservice:1.0") - assert.Len(t, module.Nodes, 7) + module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + assert.Len(t, module.Nodes, 1) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, module.Nodes, "junit:junit:4.11") + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "junit:junit:4.11") assert.Len(t, directDependency.Nodes, 1) // Check transitive dependency @@ -40,16 +44,12 @@ func TestGradleTreesWithConfig(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildGradleDependencyTree(false, false, false, nil) + modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 3) - - // Check module - module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test.gradle.publish:webservice:1.0-SNAPSHOT") - assert.Len(t, module.Nodes, 7) + assert.Len(t, modulesDependencyTrees, 7) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, module.Nodes, "org.apache.wicket:wicket:1.3.7") + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") assert.Len(t, directDependency.Nodes, 1) // Check transitive dependency @@ -64,15 +64,12 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildGradleDependencyTree(true, true, true, nil) + modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 5) - // Check module - module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.example.gradle:webservice:1.0") - assert.Len(t, module.Nodes, 6) + assert.Len(t, modulesDependencyTrees, 7) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, module.Nodes, "org.apache.wicket:wicket:1.3.7") + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") assert.Len(t, directDependency.Nodes, 1) // Check transitive dependency @@ -93,3 +90,151 @@ func TestIsGradleWrapperExist(t *testing.T) { assert.NoError(t, err) assert.True(t, isWrapperExist) } + +func TestGetDepTreeArtifactoryRepository(t *testing.T) { + tests := []struct { + name string + remoteRepo string + server *config.ServerDetails + expectedUrl string + expectedErr string + }{ + { + name: "WithAccessToken", + remoteRepo: "my-remote-repo", + server: &config.ServerDetails{ + Url: "https://myartifactory.com", + AccessToken: "my-access-token", + }, + expectedUrl: "\n\t\tmaven {\n\t\t\turl \"my-remote-repo\"\n\t\t\tcredentials {\n\t\t\t\tusername = ''\n\t\t\t\tpassword = 'my-access-token'\n\t\t\t}\n\t\t}\n", + expectedErr: "", + }, + { + name: "WithUsernameAndPassword", + remoteRepo: "my-remote-repo", + server: &config.ServerDetails{ + Url: "https://myartifactory.com", + User: "my-username", + Password: "my-password", + }, + expectedUrl: "\n\t\tmaven {\n\t\t\turl \"my-remote-repo\"\n\t\t\tcredentials {\n\t\t\t\tusername = 'my-username'\n\t\t\t\tpassword = 'my-password'\n\t\t\t}\n\t\t}\n", + expectedErr: "", + }, + { + name: "MissingCredentials", + remoteRepo: "my-remote-repo", + server: &config.ServerDetails{ + Url: "https://myartifactory.com", + }, + expectedUrl: "", + expectedErr: "either username/password or access token must be set for https://myartifactory.com", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + url, err := getDepTreeArtifactoryRepository(test.remoteRepo, test.server) + if err != nil { + assert.Equal(t, test.expectedErr, err.Error()) + } else { + assert.Equal(t, test.expectedUrl, url) + } + }) + } +} + +func TestGetGraphFromDepTree(t *testing.T) { + testCase := struct { + name string + outputFileContent []byte + expectedResult map[string][]*services.GraphNode + }{ + name: "ValidOutputFileContent", + outputFileContent: []byte(` +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== +`), + expectedResult: map[string][]*services.GraphNode{ + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": {}, + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": {{Id: GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2"}}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": {{Id: GavPackageTypeIdentifier + "commons-io:commons-io:1.2"}}, + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": {}, + GavPackageTypeIdentifier + "junit:junit:4.11": {{Id: GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3"}}, + }, + } + + result, err := (&depTreeManager{}).getGraphFromDepTree(testCase.outputFileContent) + assert.NoError(t, err) + for _, dependency := range result { + depChild, exists := testCase.expectedResult[dependency.Id] + assert.True(t, exists) + if len(depChild) > 0 { + assert.Equal(t, depChild[0].Id, dependency.Nodes[0].Id) + } + } +} + +func TestCreateDepTreeScript(t *testing.T) { + tmpDir, err := fileutils.CreateTempDir() + assert.NoError(t, err) + defer func() { + assert.NoError(t, fileutils.RemoveTempDir(tmpDir)) + }() + currDir, err := os.Getwd() + assert.NoError(t, err) + assert.NoError(t, os.Chdir(tmpDir)) + defer func() { + assert.NoError(t, os.Chdir(currDir)) + }() + manager := &depTreeManager{} + assert.NoError(t, manager.createDepTreeScript()) + defer func() { + assert.NoError(t, os.Remove(depTreeInitFile)) + }() + content, err := os.ReadFile(depTreeInitFile) + assert.NoError(t, err) + assert.Equal(t, fmt.Sprintf(depTreeInitScript, "", ""), string(content)) + manager.depsRepo = "deps-repo" + manager.releasesRepo = "release-repo" + manager.server = &config.ServerDetails{ + ArtifactoryUrl: "https://myartifactory.com/artifactory", + AccessToken: "my-access-token", + } + assert.NoError(t, manager.createDepTreeScript()) + expectedInitScript := `initscript { + repositories { + maven { + url "https://myartifactory.com/artifactory/release-repo/artifactory/oss-releases" + credentials { + username = '' + password = 'my-access-token' + } + } + mavenCentral() + } + dependencies { + classpath 'com.jfrog:gradle-dep-tree:+' + } +} + +allprojects { + repositories { + maven { + url "https://myartifactory.com/artifactory/deps-repo" + credentials { + username = '' + password = 'my-access-token' + } + } + } + apply plugin: com.jfrog.GradleDepTree +}` + content, err = os.ReadFile(depTreeInitFile) + assert.NoError(t, err) + assert.Equal(t, expectedInitScript, string(content)) +} diff --git a/xray/audit/java/javautils.go b/xray/audit/java/javautils.go index e2293e788..9b302ffe4 100644 --- a/xray/audit/java/javautils.go +++ b/xray/audit/java/javautils.go @@ -1,6 +1,7 @@ package java import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "strconv" "time" @@ -22,7 +23,10 @@ type DependencyTreeParams struct { IgnoreConfigFile bool ExcludeTestDeps bool UseWrapper bool - JavaProps map[string]any + MvnProps map[string]any + Server *config.ServerDetails + DepsRepo string + ReleasesRepo string } func createBuildConfiguration(buildName string) (*artifactoryUtils.BuildConfiguration, func(err error)) { @@ -136,9 +140,17 @@ func hasLoop(idsAdded []string, idToAdd string) bool { func BuildDependencyTree(params *DependencyTreeParams) (modules []*services.GraphNode, err error) { if params.Tool == coreutils.Maven { - return buildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.UseWrapper, params.JavaProps) + return buildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.UseWrapper, params.MvnProps) } - return buildGradleDependencyTree(params.ExcludeTestDeps, params.UseWrapper, params.IgnoreConfigFile, params.JavaProps) + server := &config.ServerDetails{} + depsRepo := "" + releaseRepo := "" + if params.IgnoreConfigFile { + server = params.Server + depsRepo = params.DepsRepo + releaseRepo = params.ReleasesRepo + } + return buildGradleDependencyTree(params.UseWrapper, server, depsRepo, releaseRepo) } type dependencyMultimap struct { diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index d4270d6b0..f127d39c2 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -34,6 +34,7 @@ type Params struct { insecureTls bool useWrapper bool depsRepo string + releasesRepo string requirementsFile string technologies []string workingDirs []string @@ -249,49 +250,6 @@ func doAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bo return } -func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependencyTrees []*services.GraphNode, e error) { - if params.progress != nil { - params.progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) - } - switch tech { - case coreutils.Maven, coreutils.Gradle: - dependencyTrees, e = getJavaDependencyTree(params, tech) - case coreutils.Npm: - dependencyTrees, e = npm.BuildDependencyTree(params.args) - case coreutils.Yarn: - dependencyTrees, e = yarn.BuildDependencyTree() - case coreutils.Go: - dependencyTrees, e = _go.BuildDependencyTree(params.serverDetails, params.depsRepo) - case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: - dependencyTrees, e = python.BuildDependencyTree(&python.AuditPython{ - Server: params.serverDetails, - Tool: pythonutils.PythonTool(tech), - RemotePypiRepo: params.depsRepo, - PipRequirementsFile: params.requirementsFile}) - case coreutils.Nuget: - dependencyTrees, e = nuget.BuildDependencyTree() - default: - e = errors.New(string(tech) + " is currently not supported") - } - dependencyTrees, e := getTechDependencyTree(params, tech) - if e != nil { - errorList.WriteString(fmt.Sprintf("'%s' audit failed when building dependency tree:\n%s\n", tech, e.Error())) - continue - } - techResults, e := audit.Audit(dependencyTrees, params.xrayGraphScanParams, params.serverDetails, params.progress, tech) - if e != nil { - errorList.WriteString(fmt.Sprintf("'%s' audit command failed:\n%s\n", tech, e.Error())) - continue - } - results = append(results, techResults...) - isMultipleRoot = len(dependencyTrees) > 1 - } - if errorList.Len() > 0 { - err = errors.New(errorList.String()) - } - return -} - func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependencyTrees []*services.GraphNode, e error) { if params.progress != nil { params.progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) @@ -322,8 +280,8 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*services.GraphNode, error) { var javaProps map[string]any - if params.DepsRepo() != "" { - javaProps = createJavaProps(params.DepsRepo(), params.ServerDetails()) + if params.DepsRepo() != "" && tech == coreutils.Maven { + javaProps = createMvnProps(params.DepsRepo(), params.ServerDetails()) } return java.BuildDependencyTree(&java.DependencyTreeParams{ Tool: tech, @@ -331,11 +289,14 @@ func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*servic IgnoreConfigFile: params.ignoreConfigFile, ExcludeTestDeps: params.excludeTestDeps, UseWrapper: params.useWrapper, - JavaProps: javaProps, + MvnProps: javaProps, + Server: params.serverDetails, + DepsRepo: params.depsRepo, + ReleasesRepo: params.releasesRepo, }) } -func createJavaProps(depsRepo string, serverDetails *config.ServerDetails) map[string]any { +func createMvnProps(depsRepo string, serverDetails *config.ServerDetails) map[string]any { authPass := serverDetails.Password if serverDetails.AccessToken != "" { authPass = serverDetails.AccessToken diff --git a/xray/commands/testdata/gradle-example-ci-server/build.gradle b/xray/commands/testdata/gradle-example-ci-server/build.gradle index 4136548d9..79619705d 100644 --- a/xray/commands/testdata/gradle-example-ci-server/build.gradle +++ b/xray/commands/testdata/gradle-example-ci-server/build.gradle @@ -31,7 +31,3 @@ allprojects { version = '1.0' status = 'integration' } - -// Setting this property to true will make the artifactoryPublish task -// skip this module (in our case, the root module): -artifactoryPublish.skip = true diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp new file mode 100644 index 000000000..359b04df4 --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp @@ -0,0 +1,33 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "spi", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "org.apache.wicket:wicket:1.3.7": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "org.slf4j:slf4j-api:1.4.2": { + "unresolved": false, + "configurations": ["compileClasspath"], + "children": {} + } + } + }, + "org.jfrog.example.gradle:shared:1.0": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + }, + "commons-lang:commons-lang:2.4": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "commons-io:commons-io:1.2": { + "unresolved": false, + "configurations": ["compileClasspath"], + "children": {} + } + } + } + } +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy new file mode 100644 index 000000000..234210f1e --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy @@ -0,0 +1,5 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= new file mode 100644 index 000000000..234210f1e --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= @@ -0,0 +1,5 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk new file mode 100644 index 000000000..234210f1e --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk @@ -0,0 +1,5 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== new file mode 100644 index 000000000..1063e46c2 --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== @@ -0,0 +1,53 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "compileClasspath", "providedCompile", "providedRuntime", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "junit:junit:4.11": { + "unresolved": false, + "configurations": ["testCompileClasspath", "testImplementation", "testRuntimeClasspath"], + "children": { + "org.hamcrest:hamcrest-core:1.3": { + "unresolved": false, + "configurations": ["testCompileClasspath"], + "children": {} + } + } + }, + "commons-io:commons-io:1.2": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + }, + "org.apache.wicket:wicket:1.3.7": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "org.slf4j:slf4j-api:1.4.2": { + "unresolved": false, + "configurations": ["compileClasspath"], + "children": {} + } + } + }, + "org.jfrog.example.gradle:api:1.0": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + }, + "org.jfrog.example.gradle:shared:1.0": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + }, + "commons-collections:commons-collections:3.2": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + }, + "commons-lang:commons-lang:2.4": { + "unresolved": false, + "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + } + } +} \ No newline at end of file From fb91ce150b720f7f023f9bd386f2f5b342e9ced7 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Thu, 23 Mar 2023 16:12:40 +0200 Subject: [PATCH 18/48] Fix tests --- xray/audit/java/gradle_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 135763e7d..383a37f48 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -106,7 +106,7 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { Url: "https://myartifactory.com", AccessToken: "my-access-token", }, - expectedUrl: "\n\t\tmaven {\n\t\t\turl \"my-remote-repo\"\n\t\t\tcredentials {\n\t\t\t\tusername = ''\n\t\t\t\tpassword = 'my-access-token'\n\t\t\t}\n\t\t}\n", + expectedUrl: "\n\t\tmaven {\n\t\t\turl \"/my-remote-repo\"\n\t\t\tcredentials {\n\t\t\t\tusername = ''\n\t\t\t\tpassword = 'my-access-token'\n\t\t\t}\n\t\t}", expectedErr: "", }, { @@ -117,7 +117,7 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { User: "my-username", Password: "my-password", }, - expectedUrl: "\n\t\tmaven {\n\t\t\turl \"my-remote-repo\"\n\t\t\tcredentials {\n\t\t\t\tusername = 'my-username'\n\t\t\t\tpassword = 'my-password'\n\t\t\t}\n\t\t}\n", + expectedUrl: "\n\t\tmaven {\n\t\t\turl \"/my-remote-repo\"\n\t\t\tcredentials {\n\t\t\t\tusername = 'my-username'\n\t\t\t\tpassword = 'my-password'\n\t\t\t}\n\t\t}", expectedErr: "", }, { From 10da0507330e358853326671fbb64b9758357291 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Thu, 23 Mar 2023 16:26:32 +0200 Subject: [PATCH 19/48] Fix tests --- xray/audit/java/gradle.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 32ab30f54..932ce5c3a 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -151,8 +151,10 @@ func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err er if err != nil { return } - if err = os.Chmod(gradleExecPath, 0777); err != nil { - return + if dtp.useWrapper { + if err = os.Chmod(gradleExecPath, 0777); err != nil { + return + } } outputFileAbsolutePath, err := filepath.Abs(depTreeOutputFile) From 0b7d7b9f8976bb6f2ed97d997235b3f1c6322a7b Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Thu, 23 Mar 2023 16:37:42 +0200 Subject: [PATCH 20/48] Fix tests --- xray/audit/java/gradle.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 932ce5c3a..39604b0d2 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -196,16 +196,15 @@ func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*ser } func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren map[string]any) { - if currNode.NodeHasLoop() { - return - } - for gav, details := range currNodeChildren { childNode := &services.GraphNode{ Id: GavPackageTypeIdentifier + gav, Nodes: []*services.GraphNode{}, Parent: currNode, } + if currNode.NodeHasLoop() { + return + } childNodeChildren := details.(map[string]any)["children"].(map[string]any) populateGradleDependencyTree(childNode, childNodeChildren) currNode.Nodes = append(currNode.Nodes, childNode) From c53e3cc1005fc2f34e6be55e2979840a6b356ea3 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Sun, 26 Mar 2023 08:37:46 +0300 Subject: [PATCH 21/48] map refactor --- xray/audit/java/gradle.go | 62 +++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 39604b0d2..016611cc7 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -45,32 +45,29 @@ allprojects { }` ) -type gradleDepsMap map[string]any - -func (gdg *gradleDepsMap) appendTree(jsonDepTree []byte) error { - var rootNode map[string]any - if err := json.Unmarshal(jsonDepTree, &rootNode); err != nil { - return err - } +type depTreeManager struct { + DependenciesMap + server *config.ServerDetails + releasesRepo string + depsRepo string + useWrapper bool +} - for gav, node := range rootNode["children"].(map[string]any) { - if _, exists := (*gdg)[gav]; !exists { - (*gdg)[gav] = node - } - } - return nil +// DependenciesMap key in this map is the "GAV" of a dependency, and the value is its children dependencies. +type DependenciesMap struct { + Dependencies map[string]DependenciesMap `json:"children"` } // The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in gradleDepsMap struct. -func (gdg *gradleDepsMap) parseDepTreeFiles(jsonFiles []byte) error { +// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in DependenciesMap struct. +func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") for _, path := range outputFiles { tree, err := os.ReadFile(strings.TrimSpace(path)) if err != nil { return err } - if err = gdg.appendTree(tree); err != nil { + if err = dtp.appendDependenciesTree(tree); err != nil { return err } @@ -78,11 +75,20 @@ func (gdg *gradleDepsMap) parseDepTreeFiles(jsonFiles []byte) error { return nil } -type depTreeManager struct { - server *config.ServerDetails - releasesRepo string - depsRepo string - useWrapper bool +func (dtp *depTreeManager) appendDependenciesTree(jsonDepTree []byte) error { + var deps DependenciesMap + if err := json.Unmarshal(jsonDepTree, &deps); err != nil { + return err + } + for gav, children := range deps.Dependencies { + if dtp.Dependencies == nil { + dtp.Dependencies = make(map[string]DependenciesMap) + } + if _, exists := dtp.Dependencies[gav]; !exists { + dtp.Dependencies[gav] = children + } + } + return nil } func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, depsRepo, releasesRepo string) (dependencyTree []*services.GraphNode, err error) { @@ -178,25 +184,24 @@ func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err er // Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*services.GraphNode, error) { - dependencyMap := gradleDepsMap{} - if err := dependencyMap.parseDepTreeFiles(outputFileContent); err != nil { + if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { return nil, err } var depsGraph []*services.GraphNode - for dependency, dependencyDetails := range dependencyMap { + for dependency, children := range dtp.Dependencies { directDependency := &services.GraphNode{ Id: GavPackageTypeIdentifier + dependency, Nodes: []*services.GraphNode{}, } - populateGradleDependencyTree(directDependency, dependencyDetails.(map[string]any)["children"].(map[string]any)) + populateGradleDependencyTree(directDependency, children) depsGraph = append(depsGraph, directDependency) } return depsGraph, nil } -func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren map[string]any) { - for gav, details := range currNodeChildren { +func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren DependenciesMap) { + for gav, children := range currNodeChildren.Dependencies { childNode := &services.GraphNode{ Id: GavPackageTypeIdentifier + gav, Nodes: []*services.GraphNode{}, @@ -205,8 +210,7 @@ func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren if currNode.NodeHasLoop() { return } - childNodeChildren := details.(map[string]any)["children"].(map[string]any) - populateGradleDependencyTree(childNode, childNodeChildren) + populateGradleDependencyTree(childNode, children) currNode.Nodes = append(currNode.Nodes, childNode) } } From fd34972cee26d9ad643baada9add574f93293a2e Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Sun, 26 Mar 2023 16:40:19 +0300 Subject: [PATCH 22/48] "Flatten" the graph --- go.mod | 2 +- go.sum | 4 +-- xray/audit/java/gradle.go | 58 +++++++++++++++++++-------------------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 43687b943..f9fa31551 100644 --- a/go.mod +++ b/go.mod @@ -96,4 +96,4 @@ require ( replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb -// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e +replace github.com/jfrog/gofrog => github.com/omerzi/gofrog v1.1.3-0.20230326133640-ef71b383b890 diff --git a/go.sum b/go.sum index 62163aca8..5bb93e609 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,6 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= -github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jfrog/jfrog-client-go v1.27.0 h1:febOOMZ7yisfMjEXCryOvcK+tMzDz591L08Jn0OeMt8= github.com/jfrog/jfrog-client-go v1.27.0/go.mod h1:dlCAYLiZX3aoM18NTFmLLkoKRXzQEIxNJN/2GXnh+qs= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -255,6 +253,8 @@ github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb h1:XWo+nYnmGE4Dy0K85oENHJcogIHpMqB5TtBkqWZy9O4= github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= +github.com/omerzi/gofrog v1.1.3-0.20230326133640-ef71b383b890 h1:uuhLY7HQcsZ+vehxWpxZbl2tngCzfcqBjStRM7zQLUw= +github.com/omerzi/gofrog v1.1.3-0.20230326133640-ef71b383b890/go.mod h1:mEv+6DWIMS7YKlDXtj9Qd0mE2KjCgs6SmaltcWKePXU= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= github.com/owenrumney/go-sarif/v2 v2.1.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 016611cc7..d794e6184 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/jfrog/build-info-go/build" + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -53,9 +54,14 @@ type depTreeManager struct { useWrapper bool } -// DependenciesMap key in this map is the "GAV" of a dependency, and the value is its children dependencies. +// DependencyPath represents a map of direct dependencies and their descendants path. +type DependencyPath struct { + Path map[string]DependencyPath `json:"children"` +} + +// DependenciesMap represents a map of direct dependencies and their descendants path from possible different projects. type DependenciesMap struct { - Dependencies map[string]DependenciesMap `json:"children"` + Dependencies map[string][]DependencyPath } // The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. @@ -76,17 +82,15 @@ func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { } func (dtp *depTreeManager) appendDependenciesTree(jsonDepTree []byte) error { - var deps DependenciesMap - if err := json.Unmarshal(jsonDepTree, &deps); err != nil { + var currentProjectDeps DependencyPath + if err := json.Unmarshal(jsonDepTree, ¤tProjectDeps); err != nil { return err } - for gav, children := range deps.Dependencies { - if dtp.Dependencies == nil { - dtp.Dependencies = make(map[string]DependenciesMap) - } - if _, exists := dtp.Dependencies[gav]; !exists { - dtp.Dependencies[gav] = children - } + if dtp.Dependencies == nil { + dtp.Dependencies = make(map[string][]DependencyPath) + } + for gav, children := range currentProjectDeps.Path { + dtp.Dependencies[gav] = append(dtp.Dependencies[gav], children) } return nil } @@ -188,30 +192,26 @@ func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*ser return nil, err } - var depsGraph []*services.GraphNode + depsSet := datastructures.MakeSet[string]() for dependency, children := range dtp.Dependencies { - directDependency := &services.GraphNode{ - Id: GavPackageTypeIdentifier + dependency, - Nodes: []*services.GraphNode{}, - } - populateGradleDependencyTree(directDependency, children) - depsGraph = append(depsGraph, directDependency) + depsSet.Add(GavPackageTypeIdentifier + dependency) + populateGradleDependencyTree(depsSet, children...) + } + + var depsGraph []*services.GraphNode + for dependency := range depsSet.Elements() { + node := &services.GraphNode{Id: dependency} + depsGraph = append(depsGraph, node) } return depsGraph, nil } -func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren DependenciesMap) { - for gav, children := range currNodeChildren.Dependencies { - childNode := &services.GraphNode{ - Id: GavPackageTypeIdentifier + gav, - Nodes: []*services.GraphNode{}, - Parent: currNode, - } - if currNode.NodeHasLoop() { - return +func populateGradleDependencyTree(depsSet *datastructures.Set[string], currNodeChildren ...DependencyPath) { + for _, currChildren := range currNodeChildren { + for childGav, descendants := range currChildren.Path { + depsSet.Add(GavPackageTypeIdentifier + childGav) + populateGradleDependencyTree(depsSet, descendants) } - populateGradleDependencyTree(childNode, children) - currNode.Nodes = append(currNode.Nodes, childNode) } } From 5359f395f7f325a16f2fbe16e104c4ee9678b657 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Mar 2023 08:32:50 +0300 Subject: [PATCH 23/48] Fix tests --- utils/coreutils/techutils.go | 4 +-- xray/audit/java/gradle_test.go | 52 +++++++++++----------------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 3620902c0..8ea6bcf81 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -58,9 +58,9 @@ var technologiesData = map[Technology]TechData{ execCommand: "mvn", }, Gradle: { - indicators: []string{".gradle"}, + indicators: []string{".gradle", ".gradle.kts"}, ciSetupSupport: true, - packageDescriptor: "build.gradle", + packageDescriptor: "build.gradle, build.gradle.kts", }, Npm: { indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 383a37f48..4cbeef01a 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -23,17 +23,10 @@ func TestGradleTreesWithoutConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(false, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 7) - // Check module - module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - assert.Len(t, module.Nodes, 1) - - // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "junit:junit:4.11") - assert.Len(t, directDependency.Nodes, 1) - - // Check transitive dependency - audit.GetAndAssertNode(t, directDependency.Nodes, "org.hamcrest:hamcrest-core:1.3") + assert.Len(t, modulesDependencyTrees, 9) + audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + audit.GetAndAssertNode(t, modulesDependencyTrees, "junit:junit:4.11") + audit.GetAndAssertNode(t, modulesDependencyTrees, "org.hamcrest:hamcrest-core:1.3") } } @@ -46,14 +39,9 @@ func TestGradleTreesWithConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 7) - - // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - assert.Len(t, directDependency.Nodes, 1) - - // Check transitive dependency - audit.GetAndAssertNode(t, directDependency.Nodes, "org.slf4j:slf4j-api:1.4.2") + assert.Len(t, modulesDependencyTrees, 8) + audit.GetAndAssertNode(t, modulesDependencyTrees, "commons-lang:commons-lang:2.4") + audit.GetAndAssertNode(t, modulesDependencyTrees, "org.slf4j:slf4j-api:1.4.2") } } @@ -66,14 +54,9 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 7) - - // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - assert.Len(t, directDependency.Nodes, 1) - - // Check transitive dependency - audit.GetAndAssertNode(t, directDependency.Nodes, "org.slf4j:slf4j-api:1.4.2") + assert.Len(t, modulesDependencyTrees, 9) + audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + audit.GetAndAssertNode(t, modulesDependencyTrees, "org.slf4j:slf4j-api:1.4.2") } } @@ -158,24 +141,23 @@ func TestGetGraphFromDepTree(t *testing.T) { ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== `), expectedResult: map[string][]*services.GraphNode{ - GavPackageTypeIdentifier + "commons-io:commons-io:1.2": {}, GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": {}, - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": {{Id: GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2"}}, + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": {}, GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": {{Id: GavPackageTypeIdentifier + "commons-io:commons-io:1.2"}}, + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": {}, GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": {}, - GavPackageTypeIdentifier + "junit:junit:4.11": {{Id: GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3"}}, + GavPackageTypeIdentifier + "junit:junit:4.11": {}, + GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2": {}, + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": {}, + GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3": {}, }, } result, err := (&depTreeManager{}).getGraphFromDepTree(testCase.outputFileContent) assert.NoError(t, err) for _, dependency := range result { - depChild, exists := testCase.expectedResult[dependency.Id] + _, exists := testCase.expectedResult[dependency.Id] assert.True(t, exists) - if len(depChild) > 0 { - assert.Equal(t, depChild[0].Id, dependency.Nodes[0].Id) - } } } From 6c452c8b3e55dc0980b081bb9ba2e72554e26276 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Mar 2023 12:04:24 +0300 Subject: [PATCH 24/48] improvements --- go.mod | 2 +- go.sum | 4 ++-- xray/audit/java/gradle.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f9fa31551..5dd1f5b25 100644 --- a/go.mod +++ b/go.mod @@ -96,4 +96,4 @@ require ( replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb -replace github.com/jfrog/gofrog => github.com/omerzi/gofrog v1.1.3-0.20230326133640-ef71b383b890 +//replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20221229104702-6b9e00fe7b36 diff --git a/go.sum b/go.sum index 5bb93e609..62163aca8 100644 --- a/go.sum +++ b/go.sum @@ -197,6 +197,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= +github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jfrog/jfrog-client-go v1.27.0 h1:febOOMZ7yisfMjEXCryOvcK+tMzDz591L08Jn0OeMt8= github.com/jfrog/jfrog-client-go v1.27.0/go.mod h1:dlCAYLiZX3aoM18NTFmLLkoKRXzQEIxNJN/2GXnh+qs= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -253,8 +255,6 @@ github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb h1:XWo+nYnmGE4Dy0K85oENHJcogIHpMqB5TtBkqWZy9O4= github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= -github.com/omerzi/gofrog v1.1.3-0.20230326133640-ef71b383b890 h1:uuhLY7HQcsZ+vehxWpxZbl2tngCzfcqBjStRM7zQLUw= -github.com/omerzi/gofrog v1.1.3-0.20230326133640-ef71b383b890/go.mod h1:mEv+6DWIMS7YKlDXtj9Qd0mE2KjCgs6SmaltcWKePXU= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= github.com/owenrumney/go-sarif/v2 v2.1.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index d794e6184..793af9af9 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -199,7 +199,7 @@ func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*ser } var depsGraph []*services.GraphNode - for dependency := range depsSet.Elements() { + for _, dependency := range depsSet.ToSlice() { node := &services.GraphNode{Id: dependency} depsGraph = append(depsGraph, node) } From 0dd7ef9c4f9b8d280c053579df78249a11e086d9 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Mar 2023 15:43:30 +0300 Subject: [PATCH 25/48] Revert "map refactor" This reverts commit c53e3cc1 --- xray/audit/java/gradle.go | 86 +++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 793af9af9..39604b0d2 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "github.com/jfrog/build-info-go/build" - "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -46,34 +45,32 @@ allprojects { }` ) -type depTreeManager struct { - DependenciesMap - server *config.ServerDetails - releasesRepo string - depsRepo string - useWrapper bool -} +type gradleDepsMap map[string]any -// DependencyPath represents a map of direct dependencies and their descendants path. -type DependencyPath struct { - Path map[string]DependencyPath `json:"children"` -} +func (gdg *gradleDepsMap) appendTree(jsonDepTree []byte) error { + var rootNode map[string]any + if err := json.Unmarshal(jsonDepTree, &rootNode); err != nil { + return err + } -// DependenciesMap represents a map of direct dependencies and their descendants path from possible different projects. -type DependenciesMap struct { - Dependencies map[string][]DependencyPath + for gav, node := range rootNode["children"].(map[string]any) { + if _, exists := (*gdg)[gav]; !exists { + (*gdg)[gav] = node + } + } + return nil } // The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in DependenciesMap struct. -func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { +// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in gradleDepsMap struct. +func (gdg *gradleDepsMap) parseDepTreeFiles(jsonFiles []byte) error { outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") for _, path := range outputFiles { tree, err := os.ReadFile(strings.TrimSpace(path)) if err != nil { return err } - if err = dtp.appendDependenciesTree(tree); err != nil { + if err = gdg.appendTree(tree); err != nil { return err } @@ -81,18 +78,11 @@ func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { return nil } -func (dtp *depTreeManager) appendDependenciesTree(jsonDepTree []byte) error { - var currentProjectDeps DependencyPath - if err := json.Unmarshal(jsonDepTree, ¤tProjectDeps); err != nil { - return err - } - if dtp.Dependencies == nil { - dtp.Dependencies = make(map[string][]DependencyPath) - } - for gav, children := range currentProjectDeps.Path { - dtp.Dependencies[gav] = append(dtp.Dependencies[gav], children) - } - return nil +type depTreeManager struct { + server *config.ServerDetails + releasesRepo string + depsRepo string + useWrapper bool } func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, depsRepo, releasesRepo string) (dependencyTree []*services.GraphNode, err error) { @@ -188,30 +178,36 @@ func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err er // Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*services.GraphNode, error) { - if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { + dependencyMap := gradleDepsMap{} + if err := dependencyMap.parseDepTreeFiles(outputFileContent); err != nil { return nil, err } - depsSet := datastructures.MakeSet[string]() - for dependency, children := range dtp.Dependencies { - depsSet.Add(GavPackageTypeIdentifier + dependency) - populateGradleDependencyTree(depsSet, children...) - } - var depsGraph []*services.GraphNode - for _, dependency := range depsSet.ToSlice() { - node := &services.GraphNode{Id: dependency} - depsGraph = append(depsGraph, node) + for dependency, dependencyDetails := range dependencyMap { + directDependency := &services.GraphNode{ + Id: GavPackageTypeIdentifier + dependency, + Nodes: []*services.GraphNode{}, + } + populateGradleDependencyTree(directDependency, dependencyDetails.(map[string]any)["children"].(map[string]any)) + depsGraph = append(depsGraph, directDependency) } return depsGraph, nil } -func populateGradleDependencyTree(depsSet *datastructures.Set[string], currNodeChildren ...DependencyPath) { - for _, currChildren := range currNodeChildren { - for childGav, descendants := range currChildren.Path { - depsSet.Add(GavPackageTypeIdentifier + childGav) - populateGradleDependencyTree(depsSet, descendants) +func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren map[string]any) { + for gav, details := range currNodeChildren { + childNode := &services.GraphNode{ + Id: GavPackageTypeIdentifier + gav, + Nodes: []*services.GraphNode{}, + Parent: currNode, + } + if currNode.NodeHasLoop() { + return } + childNodeChildren := details.(map[string]any)["children"].(map[string]any) + populateGradleDependencyTree(childNode, childNodeChildren) + currNode.Nodes = append(currNode.Nodes, childNode) } } From d2141b8b635eeb695ee87f1f2afa38b2d6fbc938 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 27 Mar 2023 08:32:50 +0300 Subject: [PATCH 26/48] Revert --- xray/audit/java/gradle.go | 69 ++++++++++++++++++++-------------- xray/audit/java/gradle_test.go | 52 ++++++++++++++++--------- 2 files changed, 75 insertions(+), 46 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 39604b0d2..40847fa5e 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -45,32 +45,36 @@ allprojects { }` ) -type gradleDepsMap map[string]any +type depTreeManager struct { + depsPaths map[string][]string + DependenciesTree + server *config.ServerDetails + releasesRepo string + depsRepo string + useWrapper bool +} -func (gdg *gradleDepsMap) appendTree(jsonDepTree []byte) error { - var rootNode map[string]any - if err := json.Unmarshal(jsonDepTree, &rootNode); err != nil { - return err - } +// DependenciesTree represents a map between dependencies to their children dependencies in multiple projects. +type DependenciesTree struct { + tree map[string][]DependenciesPaths +} - for gav, node := range rootNode["children"].(map[string]any) { - if _, exists := (*gdg)[gav]; !exists { - (*gdg)[gav] = node - } - } - return nil +// DependenciesPaths represents a map between dependencies to their children dependencies in a single project. +type DependenciesPaths struct { + Paths map[string]DependenciesPaths `json:"children"` } // The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in gradleDepsMap struct. -func (gdg *gradleDepsMap) parseDepTreeFiles(jsonFiles []byte) error { +// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in DependenciesTree struct. +func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") for _, path := range outputFiles { tree, err := os.ReadFile(strings.TrimSpace(path)) if err != nil { return err } - if err = gdg.appendTree(tree); err != nil { + + if err = dtp.appendDependenciesPaths(tree); err != nil { return err } @@ -78,11 +82,19 @@ func (gdg *gradleDepsMap) parseDepTreeFiles(jsonFiles []byte) error { return nil } -type depTreeManager struct { - server *config.ServerDetails - releasesRepo string - depsRepo string - useWrapper bool +func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte) error { + var deps DependenciesPaths + if err := json.Unmarshal(jsonDepTree, &deps); err != nil { + return err + } + if dtp.tree == nil { + dtp.tree = make(map[string][]DependenciesPaths) + } + + for gav, children := range deps.Paths { + dtp.tree[gav] = append(dtp.tree[gav], children) + } + return nil } func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, depsRepo, releasesRepo string) (dependencyTree []*services.GraphNode, err error) { @@ -178,25 +190,25 @@ func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err er // Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*services.GraphNode, error) { - dependencyMap := gradleDepsMap{} - if err := dependencyMap.parseDepTreeFiles(outputFileContent); err != nil { + if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { return nil, err } - var depsGraph []*services.GraphNode - for dependency, dependencyDetails := range dependencyMap { + for dependency, children := range dtp.tree { directDependency := &services.GraphNode{ Id: GavPackageTypeIdentifier + dependency, Nodes: []*services.GraphNode{}, } - populateGradleDependencyTree(directDependency, dependencyDetails.(map[string]any)["children"].(map[string]any)) + for _, childPath := range children { + populateGradleDependencyTree(directDependency, childPath) + } depsGraph = append(depsGraph, directDependency) } return depsGraph, nil } -func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren map[string]any) { - for gav, details := range currNodeChildren { +func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren DependenciesPaths) { + for gav, children := range currNodeChildren.Paths { childNode := &services.GraphNode{ Id: GavPackageTypeIdentifier + gav, Nodes: []*services.GraphNode{}, @@ -205,8 +217,7 @@ func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren if currNode.NodeHasLoop() { return } - childNodeChildren := details.(map[string]any)["children"].(map[string]any) - populateGradleDependencyTree(childNode, childNodeChildren) + populateGradleDependencyTree(childNode, children) currNode.Nodes = append(currNode.Nodes, childNode) } } diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 4cbeef01a..383a37f48 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -23,10 +23,17 @@ func TestGradleTreesWithoutConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(false, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 9) - audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - audit.GetAndAssertNode(t, modulesDependencyTrees, "junit:junit:4.11") - audit.GetAndAssertNode(t, modulesDependencyTrees, "org.hamcrest:hamcrest-core:1.3") + assert.Len(t, modulesDependencyTrees, 7) + // Check module + module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + assert.Len(t, module.Nodes, 1) + + // Check direct dependency + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "junit:junit:4.11") + assert.Len(t, directDependency.Nodes, 1) + + // Check transitive dependency + audit.GetAndAssertNode(t, directDependency.Nodes, "org.hamcrest:hamcrest-core:1.3") } } @@ -39,9 +46,14 @@ func TestGradleTreesWithConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 8) - audit.GetAndAssertNode(t, modulesDependencyTrees, "commons-lang:commons-lang:2.4") - audit.GetAndAssertNode(t, modulesDependencyTrees, "org.slf4j:slf4j-api:1.4.2") + assert.Len(t, modulesDependencyTrees, 7) + + // Check direct dependency + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + assert.Len(t, directDependency.Nodes, 1) + + // Check transitive dependency + audit.GetAndAssertNode(t, directDependency.Nodes, "org.slf4j:slf4j-api:1.4.2") } } @@ -54,9 +66,14 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 9) - audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - audit.GetAndAssertNode(t, modulesDependencyTrees, "org.slf4j:slf4j-api:1.4.2") + assert.Len(t, modulesDependencyTrees, 7) + + // Check direct dependency + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + assert.Len(t, directDependency.Nodes, 1) + + // Check transitive dependency + audit.GetAndAssertNode(t, directDependency.Nodes, "org.slf4j:slf4j-api:1.4.2") } } @@ -141,23 +158,24 @@ func TestGetGraphFromDepTree(t *testing.T) { ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== `), expectedResult: map[string][]*services.GraphNode{ + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": {}, GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": {}, - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": {}, + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": {{Id: GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2"}}, GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": {}, + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": {{Id: GavPackageTypeIdentifier + "commons-io:commons-io:1.2"}}, GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": {}, - GavPackageTypeIdentifier + "junit:junit:4.11": {}, - GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2": {}, - GavPackageTypeIdentifier + "commons-io:commons-io:1.2": {}, - GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3": {}, + GavPackageTypeIdentifier + "junit:junit:4.11": {{Id: GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3"}}, }, } result, err := (&depTreeManager{}).getGraphFromDepTree(testCase.outputFileContent) assert.NoError(t, err) for _, dependency := range result { - _, exists := testCase.expectedResult[dependency.Id] + depChild, exists := testCase.expectedResult[dependency.Id] assert.True(t, exists) + if len(depChild) > 0 { + assert.Equal(t, depChild[0].Id, dependency.Nodes[0].Id) + } } } From 85d330d6cfc0f0f2bdf6afda82a065ccd5abf3e1 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Wed, 29 Mar 2023 10:20:48 +0300 Subject: [PATCH 27/48] flatten graph --- go.mod | 4 ++-- go.sum | 8 ++++---- xray/audit/java/gradle.go | 18 +++++++++++------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 5dd1f5b25..a940e225b 100644 --- a/go.mod +++ b/go.mod @@ -92,8 +92,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.26.1-0.20230126120919-2cca98d435ec +replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101 -replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb +//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7 //replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20221229104702-6b9e00fe7b36 diff --git a/go.sum b/go.sum index 62163aca8..8b0d86f20 100644 --- a/go.sum +++ b/go.sum @@ -197,10 +197,10 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7 h1:kVb9X9v8LLvaA6WvY+6ainkn75+eA0vRBOrqwfPkOcw= +github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= -github.com/jfrog/jfrog-client-go v1.27.0 h1:febOOMZ7yisfMjEXCryOvcK+tMzDz591L08Jn0OeMt8= -github.com/jfrog/jfrog-client-go v1.27.0/go.mod h1:dlCAYLiZX3aoM18NTFmLLkoKRXzQEIxNJN/2GXnh+qs= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -253,8 +253,8 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb h1:XWo+nYnmGE4Dy0K85oENHJcogIHpMqB5TtBkqWZy9O4= -github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101 h1:xjoaxOWXQS45YgHvZqEfPK9qK9uj/jPQeGSD0ng4S8M= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101/go.mod h1:dlCAYLiZX3aoM18NTFmLLkoKRXzQEIxNJN/2GXnh+qs= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= github.com/owenrumney/go-sarif/v2 v2.1.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 40847fa5e..0b157ceaf 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -1,6 +1,7 @@ package java import ( + "encoding/base64" "encoding/json" "fmt" "github.com/jfrog/build-info-go/build" @@ -74,7 +75,13 @@ func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { return err } - if err = dtp.appendDependenciesPaths(tree); err != nil { + encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:] + decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName) + if err != nil { + return err + } + + if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil { return err } @@ -82,7 +89,7 @@ func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { return nil } -func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte) error { +func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error { var deps DependenciesPaths if err := json.Unmarshal(jsonDepTree, &deps); err != nil { return err @@ -90,10 +97,7 @@ func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte) error { if dtp.tree == nil { dtp.tree = make(map[string][]DependenciesPaths) } - - for gav, children := range deps.Paths { - dtp.tree[gav] = append(dtp.tree[gav], children) - } + dtp.tree[fileName] = append(dtp.tree[fileName], deps) return nil } @@ -204,7 +208,7 @@ func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*ser } depsGraph = append(depsGraph, directDependency) } - return depsGraph, nil + return services.FlattenGraph(depsGraph), nil } func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren DependenciesPaths) { From 6993b1c86a7c5805764dea1a2603004fdf8bfd85 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Wed, 29 Mar 2023 16:12:19 +0300 Subject: [PATCH 28/48] flatten graph --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a940e225b..3f64283b3 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,6 @@ require ( replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101 -//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7 +replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb //replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20221229104702-6b9e00fe7b36 diff --git a/go.sum b/go.sum index 8b0d86f20..d19f94e5f 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,6 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7 h1:kVb9X9v8LLvaA6WvY+6ainkn75+eA0vRBOrqwfPkOcw= -github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -253,6 +251,8 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb h1:XWo+nYnmGE4Dy0K85oENHJcogIHpMqB5TtBkqWZy9O4= +github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101 h1:xjoaxOWXQS45YgHvZqEfPK9qK9uj/jPQeGSD0ng4S8M= github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101/go.mod h1:dlCAYLiZX3aoM18NTFmLLkoKRXzQEIxNJN/2GXnh+qs= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= From 1a650951efea0d56c6af6222a8e68a38bc05fecc Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Thu, 30 Mar 2023 12:13:00 +0300 Subject: [PATCH 29/48] remove flatten from gradle --- xray/audit/java/gradle.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 0b157ceaf..e1757f040 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -47,7 +47,6 @@ allprojects { ) type depTreeManager struct { - depsPaths map[string][]string DependenciesTree server *config.ServerDetails releasesRepo string @@ -208,7 +207,7 @@ func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*ser } depsGraph = append(depsGraph, directDependency) } - return services.FlattenGraph(depsGraph), nil + return depsGraph, nil } func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren DependenciesPaths) { From 4a55a7123000b75b1d26d82aba4d1c251ff27106 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Thu, 30 Mar 2023 14:33:23 +0300 Subject: [PATCH 30/48] Fix tests --- xray/audit/java/gradle.go | 2 +- xray/audit/java/gradle_test.go | 62 ++++++++++++--------- xray/commands/audit/generic/auditmanager.go | 2 +- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index e1757f040..6c4ac6bf4 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -27,7 +27,7 @@ const ( mavenCentral() } dependencies { - classpath 'com.jfrog:gradle-dep-tree:+' + classpath 'com.jfrog:gradle-dep-tree:2.2.0' } } diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 383a37f48..7311a8225 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - "github.com/jfrog/jfrog-client-go/xray/services" "os" "path/filepath" "testing" @@ -23,13 +22,13 @@ func TestGradleTreesWithoutConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(false, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 7) + assert.Len(t, modulesDependencyTrees, 5) // Check module - module := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - assert.Len(t, module.Nodes, 1) + module := audit.GetAndAssertNode(t, modulesDependencyTrees, "webservice") + assert.Len(t, module.Nodes, 7) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "junit:junit:4.11") + directDependency := audit.GetAndAssertNode(t, module.Nodes, "junit:junit:4.11") assert.Len(t, directDependency.Nodes, 1) // Check transitive dependency @@ -46,14 +45,18 @@ func TestGradleTreesWithConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 7) + assert.Len(t, modulesDependencyTrees, 5) + + // Check module + module := audit.GetAndAssertNode(t, modulesDependencyTrees, "api") + assert.Len(t, module.Nodes, 4) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") + directDependency := audit.GetAndAssertNode(t, module.Nodes, "commons-lang:commons-lang:2.4") assert.Len(t, directDependency.Nodes, 1) // Check transitive dependency - audit.GetAndAssertNode(t, directDependency.Nodes, "org.slf4j:slf4j-api:1.4.2") + audit.GetAndAssertNode(t, directDependency.Nodes, "commons-io:commons-io:1.2") } } @@ -66,14 +69,11 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "") if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 7) + assert.Len(t, modulesDependencyTrees, 5) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.apache.wicket:wicket:1.3.7") - assert.Len(t, directDependency.Nodes, 1) - - // Check transitive dependency - audit.GetAndAssertNode(t, directDependency.Nodes, "org.slf4j:slf4j-api:1.4.2") + directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "services") + assert.Empty(t, directDependency.Nodes) } } @@ -147,7 +147,7 @@ func TestGetGraphFromDepTree(t *testing.T) { testCase := struct { name string outputFileContent []byte - expectedResult map[string][]*services.GraphNode + expectedResult map[string]map[string]string }{ name: "ValidOutputFileContent", outputFileContent: []byte(` @@ -157,14 +157,24 @@ func TestGetGraphFromDepTree(t *testing.T) { ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== `), - expectedResult: map[string][]*services.GraphNode{ - GavPackageTypeIdentifier + "commons-io:commons-io:1.2": {}, - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": {}, - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": {{Id: GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2"}}, - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": {{Id: GavPackageTypeIdentifier + "commons-io:commons-io:1.2"}}, - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": {}, - GavPackageTypeIdentifier + "junit:junit:4.11": {{Id: GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3"}}, + expectedResult: map[string]map[string]string{ + GavPackageTypeIdentifier + "shared": {}, + GavPackageTypeIdentifier + "gradle-example-ci-server": {}, + GavPackageTypeIdentifier + "services": {}, + GavPackageTypeIdentifier + "webservice": { + GavPackageTypeIdentifier + "junit:junit:4.11": "", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", + }, + GavPackageTypeIdentifier + "api": { + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + }, }, } @@ -173,9 +183,7 @@ func TestGetGraphFromDepTree(t *testing.T) { for _, dependency := range result { depChild, exists := testCase.expectedResult[dependency.Id] assert.True(t, exists) - if len(depChild) > 0 { - assert.Equal(t, depChild[0].Id, dependency.Nodes[0].Id) - } + assert.Equal(t, len(depChild), len(dependency.Nodes)) } } @@ -218,7 +226,7 @@ func TestCreateDepTreeScript(t *testing.T) { mavenCentral() } dependencies { - classpath 'com.jfrog:gradle-dep-tree:+' + classpath 'com.jfrog:gradle-dep-tree:2.2.0' } } diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index f127d39c2..71cb63e6d 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -275,7 +275,7 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc e = errorutils.CheckError(fmt.Errorf("%s is currently not supported", string(tech))) } - return dependencyTrees, e + return services.FlattenGraph(dependencyTrees), e } func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*services.GraphNode, error) { From a2fa58931d4d10d6e0660d312eb55321276a4401 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 12:04:21 +0300 Subject: [PATCH 31/48] Improvements --- go.mod | 10 +- go.sum | 17 ++- xray/audit/commonutils.go | 89 ++++++++++++++ xray/audit/commonutils_test.go | 126 ++++++++++++++++++++ xray/commands/audit/generic/auditmanager.go | 3 + 5 files changed, 231 insertions(+), 14 deletions(-) create mode 100644 xray/audit/commonutils_test.go diff --git a/go.mod b/go.mod index 3f64283b3..a7f4afa3b 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/forPelevin/gomoji v1.1.8 github.com/gocarina/gocsv v0.0.0-20230226133904-70c27cb2918a github.com/google/uuid v1.3.0 - github.com/gookit/color v1.5.2 + github.com/gookit/color v1.5.3 github.com/jedib0t/go-pretty/v6 v6.4.6 - github.com/jfrog/build-info-go v1.8.10 + github.com/jfrog/build-info-go v1.9.0 github.com/jfrog/gofrog v1.2.5 github.com/jfrog/jfrog-client-go v1.27.0 github.com/magiconair/properties v1.8.7 @@ -22,7 +22,7 @@ require ( github.com/stretchr/testify v1.8.2 github.com/urfave/cli v1.22.12 github.com/vbauerster/mpb/v7 v7.5.3 - golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/mod v0.9.0 golang.org/x/term v0.6.0 golang.org/x/text v0.8.0 @@ -92,8 +92,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101 +replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc -replace github.com/jfrog/build-info-go => github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e //replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20221229104702-6b9e00fe7b36 diff --git a/go.sum b/go.sum index d19f94e5f..764269e15 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI= -github.com/gookit/color v1.5.2/go.mod h1:w8h4bGiHeeBpvQVePTutdbERIUf3oJE5lZ8HM0UgXyg= +github.com/gookit/color v1.5.3 h1:twfIhZs4QLCtimkP7MOxlF3A0U/5cDPseRT9M/+2SCE= +github.com/gookit/color v1.5.3/go.mod h1:NUzwzeehUfl7GIb36pqId+UGmRfQcU/WiiyTTeNjHtE= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -197,6 +197,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e h1:MB5u0Kbq9IIDIlnpDYUq+SHTZ+/ossTOCK7z/t8S0l8= +github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e/go.mod h1:HIrpwf4p4XHpAx+N+rb8SX9yrWYWs7X4rT/s0GOJfW8= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -251,10 +253,8 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb h1:XWo+nYnmGE4Dy0K85oENHJcogIHpMqB5TtBkqWZy9O4= -github.com/omerzi/build-info-go v1.3.1-0.20230323122824-61c25bac40eb/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= -github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101 h1:xjoaxOWXQS45YgHvZqEfPK9qK9uj/jPQeGSD0ng4S8M= -github.com/omerzi/jfrog-client-go v1.13.2-0.20230329130032-38516ece5101/go.mod h1:dlCAYLiZX3aoM18NTFmLLkoKRXzQEIxNJN/2GXnh+qs= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc h1:wkzRoGOdjNULHjCSjNen8GOxuoQgx4sF3K5FHkGqDPg= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc/go.mod h1:ir1HEkCwISLjwbMmXBjbOrFqk6InFREPjbvg/5VKOJI= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.2 h1:PMDK7tXShJ9zsB7bfvlpADH5NEw1dfA9xwU8Xtdj73U= github.com/owenrumney/go-sarif/v2 v2.1.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= @@ -365,8 +365,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= -golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -495,7 +495,6 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/xray/audit/commonutils.go b/xray/audit/commonutils.go index f568d111c..807c5426b 100644 --- a/xray/audit/commonutils.go +++ b/xray/audit/commonutils.go @@ -134,3 +134,92 @@ func GetExecutableVersion(executable string) (version string, err error) { log.Debug(fmt.Sprintf("Used %q version: %s", executable, version)) return } + +func BuildImpactPaths(scanResult []services.ScanResponse, dependencyTrees []*services.GraphNode) { + for _, result := range scanResult { + if len(result.Vulnerabilities) > 0 { + buildVulnerabilitiesImpactPaths(result.Vulnerabilities, dependencyTrees) + } + if len(result.Violations) > 0 { + buildViolationsImpactPaths(result.Violations, dependencyTrees) + } + if len(result.Licenses) > 0 { + buildLicensesImpactPaths(result.Licenses, dependencyTrees) + } + } + return +} + +func buildVulnerabilitiesImpactPaths(vulnerabilities []services.Vulnerability, dependencyTrees []*services.GraphNode) { + vulnerabilitiesMap := setVulnerabilitiesPathsMap(vulnerabilities, dependencyTrees) + for i := range vulnerabilities { + for dependencyName := range vulnerabilities[i].Components { + updateVulnerableComponent(vulnerabilities[i].Components, vulnerabilitiesMap[dependencyName], dependencyName) + } + } +} + +func buildViolationsImpactPaths(violations []services.Violation, dependencyTrees []*services.GraphNode) { + violationsMap := setVulnerabilitiesPathsMap(violations, dependencyTrees) + for i := range violations { + for dependencyName := range violations[i].Components { + updateVulnerableComponent(violations[i].Components, violationsMap[dependencyName], dependencyName) + } + } +} + +func buildLicensesImpactPaths(licenses []services.License, dependencyTrees []*services.GraphNode) { + licensesMap := setVulnerabilitiesPathsMap(licenses, dependencyTrees) + for i := range licenses { + for dependencyName := range licenses[i].Components { + updateVulnerableComponent(licenses[i].Components, licensesMap[dependencyName], dependencyName) + } + } +} + +func setVulnerabilitiesPathsMap(issues interface{}, dependencyTrees []*services.GraphNode) map[string][][]services.ImpactPathNode { + issueMap := make(map[string][][]services.ImpactPathNode) + switch v := issues.(type) { + case []services.Vulnerability: + for _, vulnerability := range v { + for dependencyName := range vulnerability.Components { + issueMap[dependencyName] = [][]services.ImpactPathNode{} + } + } + case []services.Violation: + for _, violation := range v { + for dependencyName := range violation.Components { + issueMap[dependencyName] = [][]services.ImpactPathNode{} + } + } + case []services.License: + for _, license := range v { + for dependencyName := range license.Components { + issueMap[dependencyName] = [][]services.ImpactPathNode{} + } + } + } + + for _, dependency := range dependencyTrees { + setPathsForIssues(dependency, issueMap, []services.ImpactPathNode{}) + } + return issueMap +} + +func updateVulnerableComponent(components map[string]services.Component, impactPaths [][]services.ImpactPathNode, dependencyName string) { + components[dependencyName] = services.Component{ + FixedVersions: components[dependencyName].FixedVersions, + ImpactPaths: impactPaths, + Cpes: components[dependencyName].Cpes, + } +} + +func setPathsForIssues(dependency *services.GraphNode, issuesMap map[string][][]services.ImpactPathNode, impactPath []services.ImpactPathNode) { + impactPath = append(impactPath, services.ImpactPathNode{ComponentId: dependency.Id}) + if _, exists := issuesMap[dependency.Id]; exists { + issuesMap[dependency.Id] = append(issuesMap[dependency.Id], impactPath) + } + for _, depChild := range dependency.Nodes { + setPathsForIssues(depChild, issuesMap, impactPath) + } +} diff --git a/xray/audit/commonutils_test.go b/xray/audit/commonutils_test.go new file mode 100644 index 000000000..6f455df18 --- /dev/null +++ b/xray/audit/commonutils_test.go @@ -0,0 +1,126 @@ +package audit + +import ( + "github.com/jfrog/jfrog-client-go/xray/services" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSetPathsForIssues(t *testing.T) { + // Create a test dependency tree + rootNode := &services.GraphNode{Id: "root"} + childNode1 := &services.GraphNode{Id: "child1"} + childNode2 := &services.GraphNode{Id: "child2"} + childNode3 := &services.GraphNode{Id: "child3"} + childNode4 := &services.GraphNode{Id: "child4"} + childNode5 := &services.GraphNode{Id: "child5"} + rootNode.Nodes = []*services.GraphNode{childNode1, childNode2, childNode3} + childNode2.Nodes = []*services.GraphNode{childNode4} + childNode3.Nodes = []*services.GraphNode{childNode5} + + // Create a test issues map + issuesMap := make(map[string][][]services.ImpactPathNode) + issuesMap["child1"] = make([][]services.ImpactPathNode, 0) + issuesMap["child4"] = make([][]services.ImpactPathNode, 0) + issuesMap["child5"] = make([][]services.ImpactPathNode, 0) + + // Call setPathsForIssues with the test data + setPathsForIssues(rootNode, issuesMap, []services.ImpactPathNode{}) + + // Check the results + assert.Equal(t, issuesMap["child1"][0][0].ComponentId, "root") + assert.Equal(t, issuesMap["child1"][0][1].ComponentId, "child1") + + assert.Equal(t, issuesMap["child4"][0][0].ComponentId, "root") + assert.Equal(t, issuesMap["child4"][0][1].ComponentId, "child2") + assert.Equal(t, issuesMap["child4"][0][2].ComponentId, "child4") + + assert.Equal(t, issuesMap["child5"][0][0].ComponentId, "root") + assert.Equal(t, issuesMap["child5"][0][1].ComponentId, "child3") + assert.Equal(t, issuesMap["child5"][0][2].ComponentId, "child5") +} + +func TestUpdateVulnerableComponent(t *testing.T) { + // Create test data + components := map[string]services.Component{ + "dependency1": { + FixedVersions: []string{"1.0.0"}, + ImpactPaths: [][]services.ImpactPathNode{}, + }, + } + impactPaths := [][]services.ImpactPathNode{ + {{ComponentId: "dependency2"}}, + } + dependencyName := "dependency1" + updateVulnerableComponent(components, impactPaths, dependencyName) + + // Check the result + expected := services.Component{ + FixedVersions: []string{"1.0.0"}, + ImpactPaths: impactPaths, + } + assert.Equal(t, expected, components[dependencyName]) +} + +func TestBuildImpactPaths(t *testing.T) { + // create sample scan result and dependency trees + scanResult := []services.ScanResponse{ + { + Vulnerabilities: []services.Vulnerability{ + { + Components: map[string]services.Component{ + "dep1": { + FixedVersions: []string{"1.2.3"}, + Cpes: []string{"cpe:/o:vendor:product:1.2.3"}, + }, + }, + }, + }, + Violations: []services.Violation{ + { + Components: map[string]services.Component{ + "dep2": { + FixedVersions: []string{"4.5.6"}, + Cpes: []string{"cpe:/o:vendor:product:4.5.6"}, + }, + }, + }, + }, + Licenses: []services.License{ + { + Components: map[string]services.Component{ + "dep3": { + FixedVersions: []string{"7.8.9"}, + Cpes: []string{"cpe:/o:vendor:product:7.8.9"}, + }, + }, + }, + }, + }, + } + dependencyTrees := []*services.GraphNode{ + { + Id: "dep1", + Nodes: []*services.GraphNode{ + { + Id: "dep2", + Nodes: []*services.GraphNode{ + { + Id: "dep3", + Nodes: []*services.GraphNode{}, + }, + }, + }, + }, + }, + } + + BuildImpactPaths(scanResult, dependencyTrees) + // assert that the components were updated with impact paths + expectedImpactPaths := [][]services.ImpactPathNode{{{ComponentId: "dep1"}}} + assert.Equal(t, expectedImpactPaths, scanResult[0].Vulnerabilities[0].Components["dep1"].ImpactPaths) + expectedImpactPaths = [][]services.ImpactPathNode{{{ComponentId: "dep1"}, {ComponentId: "dep2"}}} + assert.Equal(t, expectedImpactPaths, scanResult[0].Violations[0].Components["dep2"].ImpactPaths) + expectedImpactPaths = [][]services.ImpactPathNode{{{ComponentId: "dep1"}, {ComponentId: "dep2"}, {ComponentId: "dep3"}}} + assert.Equal(t, expectedImpactPaths, scanResult[0].Licenses[0].Components["dep3"].ImpactPaths) +} diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 71cb63e6d..be2488d2a 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -29,6 +29,7 @@ type Params struct { xrayGraphScanParams services.XrayGraphScanParams serverDetails *config.ServerDetails progress ioUtils.ProgressMgr + dependencyTrees []*services.GraphNode ignoreConfigFile bool excludeTestDeps bool insecureTls bool @@ -237,6 +238,7 @@ func doAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bo continue } techResults, e := audit.Audit(dependencyTrees, params.xrayGraphScanParams, params.serverDetails, params.progress, tech) + audit.BuildImpactPaths(techResults, params.dependencyTrees) if e != nil { errorList.WriteString(fmt.Sprintf("'%s' audit command failed:\n%s\n", tech, e.Error())) continue @@ -275,6 +277,7 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc e = errorutils.CheckError(fmt.Errorf("%s is currently not supported", string(tech))) } + params.dependencyTrees = dependencyTrees return services.FlattenGraph(dependencyTrees), e } From c9f48fad4cf19e1e6d922d282c005f627cafd03e Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 12:12:59 +0300 Subject: [PATCH 32/48] Merge remote-tracking branch 'upstream/dev' into audit-with-gradledeptree # Conflicts: # go.sum --- xray/audit/commonutils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/xray/audit/commonutils.go b/xray/audit/commonutils.go index 807c5426b..c6f0c310a 100644 --- a/xray/audit/commonutils.go +++ b/xray/audit/commonutils.go @@ -147,7 +147,6 @@ func BuildImpactPaths(scanResult []services.ScanResponse, dependencyTrees []*ser buildLicensesImpactPaths(result.Licenses, dependencyTrees) } } - return } func buildVulnerabilitiesImpactPaths(vulnerabilities []services.Vulnerability, dependencyTrees []*services.GraphNode) { From 3b55c33c4b2a785d9ed91b2c2c5bfbc139f96718 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 12:29:58 +0300 Subject: [PATCH 33/48] Merge remote-tracking branch 'upstream/dev' into audit-with-gradledeptree # Conflicts: # go.sum --- xray/audit/java/gradle_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 7311a8225..35bef3e73 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -151,10 +151,10 @@ func TestGetGraphFromDepTree(t *testing.T) { }{ name: "ValidOutputFileContent", outputFileContent: []byte(` -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== `), expectedResult: map[string]map[string]string{ From 4a80a2bfe53c0ed7a97c105ee56f885fd15e1bf1 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 12:37:24 +0300 Subject: [PATCH 34/48] Merge remote-tracking branch 'upstream/dev' into audit-with-gradledeptree # Conflicts: # go.sum --- .../build/gradle-dep-tree/{YXBp => YXBp==} | 0 ...WV4YW1wbGUtY2ktc2VydmVy => Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy==} | 0 .../build/gradle-dep-tree/{c2hhcmVk => c2hhcmVk==} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/{YXBp => YXBp==} (100%) rename xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/{Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy => Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy==} (100%) rename xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/{c2hhcmVk => c2hhcmVk==} (100%) diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== similarity index 100% rename from xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp rename to xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== similarity index 100% rename from xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy rename to xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== similarity index 100% rename from xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk rename to xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== From 1e5a5d38f7223eed7f65a2697892f92b0fd7348b Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 12:59:26 +0300 Subject: [PATCH 35/48] Merge remote-tracking branch 'upstream/dev' into audit-with-gradledeptree # Conflicts: # go.sum --- go.mod | 8 ++--- xray/audit/java/gradle_test.go | 8 ++--- .../build/gradle-dep-tree/YXBp | 33 +++++++++++++++++++ .../build/gradle-dep-tree/YXBp== | 33 ------------------- .../Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy | 5 +++ .../Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== | 5 --- .../build/gradle-dep-tree/c2VydmljZXM= | 2 +- .../build/gradle-dep-tree/c2hhcmVk | 5 +++ .../build/gradle-dep-tree/c2hhcmVk== | 5 --- .../build/gradle-dep-tree/d2Vic2VydmljZQ== | 14 ++++---- 10 files changed, 59 insertions(+), 59 deletions(-) create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== create mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== diff --git a/go.mod b/go.mod index 99e80e612..b8a262f1c 100644 --- a/go.mod +++ b/go.mod @@ -91,9 +91,9 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc - -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e +// +//replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc +// +//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e //replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20221229104702-6b9e00fe7b36 diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 35bef3e73..95294fad3 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -151,11 +151,11 @@ func TestGetGraphFromDepTree(t *testing.T) { }{ name: "ValidOutputFileContent", outputFileContent: []byte(` -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp ../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= +../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk `), expectedResult: map[string]map[string]string{ GavPackageTypeIdentifier + "shared": {}, diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp new file mode 100644 index 000000000..fa96587ab --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp @@ -0,0 +1,33 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "spi", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], + "children": { + "org.apache.wicket:wicket:1.3.7": { + "unresolved": false, + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "org.slf4j:slf4j-api:1.4.2": { + "unresolved": false, + "configurations": ["compileClasspath"], + "children": {} + } + } + }, + "org.jfrog.example.gradle:shared:1.0": { + "unresolved": false, + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": {} + }, + "commons-lang:commons-lang:2.4": { + "unresolved": false, + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "children": { + "commons-io:commons-io:1.2": { + "unresolved": false, + "configurations": ["compileClasspath"], + "children": {} + } + } + } + } +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== deleted file mode 100644 index 359b04df4..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp== +++ /dev/null @@ -1,33 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "spi", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], - "children": { - "org.apache.wicket:wicket:1.3.7": { - "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": { - "org.slf4j:slf4j-api:1.4.2": { - "unresolved": false, - "configurations": ["compileClasspath"], - "children": {} - } - } - }, - "org.jfrog.example.gradle:shared:1.0": { - "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - }, - "commons-lang:commons-lang:2.4": { - "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": { - "commons-io:commons-io:1.2": { - "unresolved": false, - "configurations": ["compileClasspath"], - "children": {} - } - } - } - } -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy new file mode 100644 index 000000000..da46a372c --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy @@ -0,0 +1,5 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], + "children": {} +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== deleted file mode 100644 index 234210f1e..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy== +++ /dev/null @@ -1,5 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= index 234210f1e..da46a372c 100644 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= @@ -1,5 +1,5 @@ { "unresolved": false, - "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], "children": {} } \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk new file mode 100644 index 000000000..da46a372c --- /dev/null +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk @@ -0,0 +1,5 @@ +{ + "unresolved": false, + "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], + "children": {} +} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== deleted file mode 100644 index 234210f1e..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk== +++ /dev/null @@ -1,5 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "compileClasspath", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== index 1063e46c2..c2cc7d3d1 100644 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== +++ b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== @@ -1,6 +1,6 @@ { "unresolved": false, - "configurations": ["annotationProcessor", "compileClasspath", "providedCompile", "providedRuntime", "runtimeClasspath", "testAnnotationProcessor", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "providedCompile", "providedRuntime", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], "children": { "junit:junit:4.11": { "unresolved": false, @@ -15,12 +15,12 @@ }, "commons-io:commons-io:1.2": { "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], "children": {} }, "org.apache.wicket:wicket:1.3.7": { "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], "children": { "org.slf4j:slf4j-api:1.4.2": { "unresolved": false, @@ -31,22 +31,22 @@ }, "org.jfrog.example.gradle:api:1.0": { "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], "children": {} }, "org.jfrog.example.gradle:shared:1.0": { "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], "children": {} }, "commons-collections:commons-collections:3.2": { "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], "children": {} }, "commons-lang:commons-lang:2.4": { "unresolved": false, - "configurations": ["compileClasspath", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], + "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], "children": {} } } From 878097c6a28b231666117e8d0cdf239cca354415 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 13:01:54 +0300 Subject: [PATCH 36/48] Merge remote-tracking branch 'upstream/dev' into audit-with-gradledeptree # Conflicts: # go.sum --- go.mod | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b8a262f1c..99e80e612 100644 --- a/go.mod +++ b/go.mod @@ -91,9 +91,9 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -// -//replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc -// -//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e + +replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230403073742-d41800fc68cc + +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e //replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20221229104702-6b9e00fe7b36 From 5a42af909d9becf6b7d61e8fa5204053175c26a3 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 13:12:05 +0300 Subject: [PATCH 37/48] debug logs --- xray/audit/java/gradle.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 6c4ac6bf4..7b97a4096 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -69,12 +69,14 @@ type DependenciesPaths struct { func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") for _, path := range outputFiles { + log.Info("reading", strings.TrimSpace(path)) tree, err := os.ReadFile(strings.TrimSpace(path)) if err != nil { return err } encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:] + log.Info("decoding", encodedFileName) decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName) if err != nil { return err From 169483f1a61ad2e167cd6bb0324781ef1ef01464 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 13:47:18 +0300 Subject: [PATCH 38/48] Fix tests --- xray/audit/java/gradle_test.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 95294fad3..544ea2194 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -144,22 +144,18 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { } func TestGetGraphFromDepTree(t *testing.T) { + // Create and change directory to test workspace + tempDirPath, cleanUp := audit.CreateTestWorkspace(t, "gradle-example-ci-server") + defer cleanUp() + assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) testCase := struct { - name string - outputFileContent []byte - expectedResult map[string]map[string]string + name string + expectedResult map[string]map[string]string }{ name: "ValidOutputFileContent", - outputFileContent: []byte(` -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= -../../commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk -`), expectedResult: map[string]map[string]string{ GavPackageTypeIdentifier + "shared": {}, - GavPackageTypeIdentifier + "gradle-example-ci-server": {}, + GavPackageTypeIdentifier + filepath.Base(tempDirPath): {}, GavPackageTypeIdentifier + "services": {}, GavPackageTypeIdentifier + "webservice": { GavPackageTypeIdentifier + "junit:junit:4.11": "", @@ -178,7 +174,10 @@ func TestGetGraphFromDepTree(t *testing.T) { }, } - result, err := (&depTreeManager{}).getGraphFromDepTree(testCase.outputFileContent) + manager := &depTreeManager{} + outputFileContent, err := manager.runGradleDepTree() + assert.NoError(t, err) + result, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent) assert.NoError(t, err) for _, dependency := range result { depChild, exists := testCase.expectedResult[dependency.Id] From 72b51ca2a1e49c982cb8390111666394970bccba Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 14:24:40 +0300 Subject: [PATCH 39/48] Fix tests --- .../build/gradle-dep-tree/YXBp | 33 ------------ .../Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy | 5 -- .../build/gradle-dep-tree/c2VydmljZXM= | 5 -- .../build/gradle-dep-tree/c2hhcmVk | 5 -- .../build/gradle-dep-tree/d2Vic2VydmljZQ== | 53 ------------------- 5 files changed, 101 deletions(-) delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk delete mode 100644 xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp deleted file mode 100644 index fa96587ab..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/YXBp +++ /dev/null @@ -1,33 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "spi", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], - "children": { - "org.apache.wicket:wicket:1.3.7": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": { - "org.slf4j:slf4j-api:1.4.2": { - "unresolved": false, - "configurations": ["compileClasspath"], - "children": {} - } - } - }, - "org.jfrog.example.gradle:shared:1.0": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - }, - "commons-lang:commons-lang:2.4": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": { - "commons-io:commons-io:1.2": { - "unresolved": false, - "configurations": ["compileClasspath"], - "children": {} - } - } - } - } -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy deleted file mode 100644 index da46a372c..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/Z3JhZGxlLWV4YW1wbGUtY2ktc2VydmVy +++ /dev/null @@ -1,5 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], - "children": {} -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= deleted file mode 100644 index da46a372c..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2VydmljZXM= +++ /dev/null @@ -1,5 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], - "children": {} -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk deleted file mode 100644 index da46a372c..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/c2hhcmVk +++ /dev/null @@ -1,5 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], - "children": {} -} \ No newline at end of file diff --git a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== b/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== deleted file mode 100644 index c2cc7d3d1..000000000 --- a/xray/commands/testdata/gradle-example-ci-server/build/gradle-dep-tree/d2Vic2VydmljZQ== +++ /dev/null @@ -1,53 +0,0 @@ -{ - "unresolved": false, - "configurations": ["annotationProcessor", "archives", "compile", "compileClasspath", "compileOnly", "default", "providedCompile", "providedRuntime", "runtime", "runtimeClasspath", "testAnnotationProcessor", "testCompile", "testCompileClasspath", "testCompileOnly", "testRuntime", "testRuntimeClasspath"], - "children": { - "junit:junit:4.11": { - "unresolved": false, - "configurations": ["testCompileClasspath", "testImplementation", "testRuntimeClasspath"], - "children": { - "org.hamcrest:hamcrest-core:1.3": { - "unresolved": false, - "configurations": ["testCompileClasspath"], - "children": {} - } - } - }, - "commons-io:commons-io:1.2": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - }, - "org.apache.wicket:wicket:1.3.7": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": { - "org.slf4j:slf4j-api:1.4.2": { - "unresolved": false, - "configurations": ["compileClasspath"], - "children": {} - } - } - }, - "org.jfrog.example.gradle:api:1.0": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - }, - "org.jfrog.example.gradle:shared:1.0": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - }, - "commons-collections:commons-collections:3.2": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - }, - "commons-lang:commons-lang:2.4": { - "unresolved": false, - "configurations": ["compileClasspath", "default", "implementation", "runtimeClasspath", "testCompileClasspath", "testRuntimeClasspath"], - "children": {} - } - } -} \ No newline at end of file From 91f4b416e35e234d766a9719ac581a557057404a Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Mon, 3 Apr 2023 17:07:27 +0300 Subject: [PATCH 40/48] add SetReleaseRepo --- xray/commands/audit/generic/auditmanager.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index be2488d2a..72b5c270b 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -159,6 +159,11 @@ func (params *Params) SetDepsRepo(depsRepo string) *Params { return params } +func (params *Params) SetReleasesRepo(releasesRepo string) *Params { + params.releasesRepo = releasesRepo + return params +} + func (params *Params) SetInstallFunc(installFunc func(tech string) error) *Params { params.installFunc = installFunc return params From 6eb6755f3e16e400b3c7d97f9549b6f7866c486d Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 09:34:31 +0300 Subject: [PATCH 41/48] CR Changes --- go.mod | 2 +- go.sum | 4 +- utils/config/config.go | 2 +- xray/audit/commonutils.go | 5 ++- xray/audit/commonutils_test.go | 2 +- xray/audit/java/gradle.go | 47 +++++++++------------ xray/audit/java/javautils.go | 4 +- xray/commands/audit/generic/auditmanager.go | 29 +++++++------ 8 files changed, 45 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 0debd45f5..3715b9455 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,6 @@ require ( replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230403150018-9014235c59f5 -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230316095417-a9f6b73206d7 +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e diff --git a/go.sum b/go.sum index 2ed2de628..f5de76539 100644 --- a/go.sum +++ b/go.sum @@ -198,8 +198,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jfrog/build-info-go v1.9.0 h1:gLxBfp4C6pVz+bKTmsqwFGZAueVMuzGw+/M9HZgtGG4= -github.com/jfrog/build-info-go v1.9.0/go.mod h1:dQ8OKddrbgtO3jK9uLYoqmRGNEjuDuNXV0bSRdpeTCI= +github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e h1:MB5u0Kbq9IIDIlnpDYUq+SHTZ+/ossTOCK7z/t8S0l8= +github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e/go.mod h1:HIrpwf4p4XHpAx+N+rb8SX9yrWYWs7X4rT/s0GOJfW8= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jfrog/jfrog-client-go v1.28.1-0.20230403150018-9014235c59f5 h1:YBkUjzhXd1GS/iK1e4PonUyTqxZtbPsrRZ/2XaPZpHo= diff --git a/utils/config/config.go b/utils/config/config.go index 3f9c3fefc..9370d2eaa 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -589,7 +589,7 @@ type MissionControlDetails struct { } func (serverDetails *ServerDetails) IsEmpty() bool { - return len(serverDetails.ServerId) == 0 + return len(serverDetails.ServerId) == 0 && serverDetails.Url == "" } func (serverDetails *ServerDetails) SetUser(username string) { diff --git a/xray/audit/commonutils.go b/xray/audit/commonutils.go index c6f0c310a..088a51e25 100644 --- a/xray/audit/commonutils.go +++ b/xray/audit/commonutils.go @@ -135,7 +135,9 @@ func GetExecutableVersion(executable string) (version string, err error) { return } -func BuildImpactPaths(scanResult []services.ScanResponse, dependencyTrees []*services.GraphNode) { +// BuildImpactPathsForScanResponse builds the full impact paths for each vulnerability found in the scanResult argument, using the dependencyTrees argument. +// Returns the updated services.ScanResponse slice. +func BuildImpactPathsForScanResponse(scanResult []services.ScanResponse, dependencyTrees []*services.GraphNode) []services.ScanResponse { for _, result := range scanResult { if len(result.Vulnerabilities) > 0 { buildVulnerabilitiesImpactPaths(result.Vulnerabilities, dependencyTrees) @@ -147,6 +149,7 @@ func BuildImpactPaths(scanResult []services.ScanResponse, dependencyTrees []*ser buildLicensesImpactPaths(result.Licenses, dependencyTrees) } } + return scanResult } func buildVulnerabilitiesImpactPaths(vulnerabilities []services.Vulnerability, dependencyTrees []*services.GraphNode) { diff --git a/xray/audit/commonutils_test.go b/xray/audit/commonutils_test.go index 6f455df18..1299d997d 100644 --- a/xray/audit/commonutils_test.go +++ b/xray/audit/commonutils_test.go @@ -115,7 +115,7 @@ func TestBuildImpactPaths(t *testing.T) { }, } - BuildImpactPaths(scanResult, dependencyTrees) + scanResult = BuildImpactPathsForScanResponse(scanResult, dependencyTrees) // assert that the components were updated with impact paths expectedImpactPaths := [][]services.ImpactPathNode{{{ComponentId: "dep1"}}} assert.Equal(t, expectedImpactPaths, scanResult[0].Vulnerabilities[0].Components["dep1"].ImpactPaths) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 7b97a4096..4e0732b14 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -8,6 +8,7 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" @@ -47,56 +48,53 @@ allprojects { ) type depTreeManager struct { - DependenciesTree + dependenciesTree server *config.ServerDetails releasesRepo string depsRepo string useWrapper bool } -// DependenciesTree represents a map between dependencies to their children dependencies in multiple projects. -type DependenciesTree struct { - tree map[string][]DependenciesPaths +// dependenciesTree represents a map between dependencies to their children dependencies in multiple projects. +type dependenciesTree struct { + tree map[string][]dependenciesPaths } -// DependenciesPaths represents a map between dependencies to their children dependencies in a single project. -type DependenciesPaths struct { - Paths map[string]DependenciesPaths `json:"children"` +// dependenciesPaths represents a map between dependencies to their children dependencies in a single project. +type dependenciesPaths struct { + Paths map[string]dependenciesPaths `json:"children"` } // The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in DependenciesTree struct. +// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in dependenciesTree struct. func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") for _, path := range outputFiles { - log.Info("reading", strings.TrimSpace(path)) tree, err := os.ReadFile(strings.TrimSpace(path)) if err != nil { - return err + return errorutils.CheckError(err) } encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:] - log.Info("decoding", encodedFileName) decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName) if err != nil { - return err + return errorutils.CheckError(err) } if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil { - return err + return errorutils.CheckError(err) } - } return nil } func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error { - var deps DependenciesPaths + var deps dependenciesPaths if err := json.Unmarshal(jsonDepTree, &deps); err != nil { - return err + return errorutils.CheckError(err) } if dtp.tree == nil { - dtp.tree = make(map[string][]DependenciesPaths) + dtp.tree = make(map[string][]dependenciesPaths) } dtp.tree[fileName] = append(dtp.tree[fileName], deps) return nil @@ -168,11 +166,6 @@ func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err er if err != nil { return } - if dtp.useWrapper { - if err = os.Chmod(gradleExecPath, 0777); err != nil { - return - } - } outputFileAbsolutePath, err := filepath.Abs(depTreeOutputFile) if err != nil { @@ -212,7 +205,7 @@ func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*ser return depsGraph, nil } -func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren DependenciesPaths) { +func populateGradleDependencyTree(currNode *services.GraphNode, currNodeChildren dependenciesPaths) { for gav, children := range currNodeChildren.Paths { childNode := &services.GraphNode{ Id: GavPackageTypeIdentifier + gav, @@ -237,21 +230,19 @@ func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDet return "", fmt.Errorf("either username/password or access token must be set for %s", server.Url) } return fmt.Sprintf(artifactoryRepository, - server.ArtifactoryUrl, + strings.TrimSuffix(server.ArtifactoryUrl, "/"), remoteRepo, user, pass), nil } +// getGradleConfig the remote repository and server details defined in the .jfrog/projects/gradle.yaml file, if configured. func getGradleConfig() (string, *config.ServerDetails, error) { var exists bool configFilePath, exists, err := utils.GetProjectConfFilePath(utils.Gradle) - if err != nil { + if err != nil || !exists { return "", nil, err } - if !exists { - return "", nil, nil - } log.Debug("Using resolver config from", configFilePath) configContent, err := utils.ReadConfigFile(configFilePath, utils.YAML) if err != nil { diff --git a/xray/audit/java/javautils.go b/xray/audit/java/javautils.go index fc1240871..856939a05 100644 --- a/xray/audit/java/javautils.go +++ b/xray/audit/java/javautils.go @@ -23,7 +23,7 @@ type DependencyTreeParams struct { IgnoreConfigFile bool ExcludeTestDeps bool UseWrapper bool - MvnProps map[string]any + JavaProps map[string]any Server *config.ServerDetails DepsRepo string ReleasesRepo string @@ -133,7 +133,7 @@ func hasLoop(idsAdded []string, idToAdd string) bool { func BuildDependencyTree(params *DependencyTreeParams) (modules []*services.GraphNode, err error) { if params.Tool == coreutils.Maven { - return buildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.UseWrapper, params.MvnProps) + return buildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.UseWrapper, params.JavaProps) } server := &config.ServerDetails{} depsRepo := "" diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 72b5c270b..0458e8f95 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -243,53 +243,54 @@ func doAudit(params *Params) (results []services.ScanResponse, isMultipleRoot bo continue } techResults, e := audit.Audit(dependencyTrees, params.xrayGraphScanParams, params.serverDetails, params.progress, tech) - audit.BuildImpactPaths(techResults, params.dependencyTrees) if e != nil { errorList.WriteString(fmt.Sprintf("'%s' audit command failed:\n%s\n", tech, e.Error())) continue } + techResults = audit.BuildImpactPathsForScanResponse(techResults, params.dependencyTrees) results = append(results, techResults...) isMultipleRoot = len(dependencyTrees) > 1 } if errorList.Len() > 0 { - err = errors.New(errorList.String()) + err = errorutils.CheckErrorf(errorList.String()) } return } -func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependencyTrees []*services.GraphNode, e error) { +func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependencyTrees []*services.GraphNode, err error) { if params.progress != nil { params.progress.SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) } switch tech { case coreutils.Maven, coreutils.Gradle: - dependencyTrees, e = getJavaDependencyTree(params, tech) + dependencyTrees, err = getJavaDependencyTree(params, tech) case coreutils.Npm: - dependencyTrees, e = npm.BuildDependencyTree(params.args) + dependencyTrees, err = npm.BuildDependencyTree(params.args) case coreutils.Yarn: - dependencyTrees, e = yarn.BuildDependencyTree() + dependencyTrees, err = yarn.BuildDependencyTree() case coreutils.Go: - dependencyTrees, e = _go.BuildDependencyTree(params.serverDetails, params.depsRepo) + dependencyTrees, err = _go.BuildDependencyTree(params.serverDetails, params.depsRepo) case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: - dependencyTrees, e = python.BuildDependencyTree(&python.AuditPython{ + dependencyTrees, err = python.BuildDependencyTree(&python.AuditPython{ Server: params.serverDetails, Tool: pythonutils.PythonTool(tech), RemotePypiRepo: params.depsRepo, PipRequirementsFile: params.requirementsFile}) case coreutils.Nuget: - dependencyTrees, e = nuget.BuildDependencyTree() + dependencyTrees, err = nuget.BuildDependencyTree() default: - e = errorutils.CheckError(fmt.Errorf("%s is currently not supported", string(tech))) + err = errorutils.CheckError(fmt.Errorf("%s is currently not supported", string(tech))) + return } params.dependencyTrees = dependencyTrees - return services.FlattenGraph(dependencyTrees), e + return services.FlattenGraph(dependencyTrees), err } func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*services.GraphNode, error) { var javaProps map[string]any if params.DepsRepo() != "" && tech == coreutils.Maven { - javaProps = createMvnProps(params.DepsRepo(), params.ServerDetails()) + javaProps = createJavaProps(params.DepsRepo(), params.ServerDetails()) } return java.BuildDependencyTree(&java.DependencyTreeParams{ Tool: tech, @@ -297,14 +298,14 @@ func getJavaDependencyTree(params *Params, tech coreutils.Technology) ([]*servic IgnoreConfigFile: params.ignoreConfigFile, ExcludeTestDeps: params.excludeTestDeps, UseWrapper: params.useWrapper, - MvnProps: javaProps, + JavaProps: javaProps, Server: params.serverDetails, DepsRepo: params.depsRepo, ReleasesRepo: params.releasesRepo, }) } -func createMvnProps(depsRepo string, serverDetails *config.ServerDetails) map[string]any { +func createJavaProps(depsRepo string, serverDetails *config.ServerDetails) map[string]any { authPass := serverDetails.Password if serverDetails.AccessToken != "" { authPass = serverDetails.AccessToken From dc02e043f34a6fcf6ca9b4c1557a7f18a1c452c9 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 11:24:59 +0300 Subject: [PATCH 42/48] Fix tests --- xray/audit/java/gradle_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 544ea2194..6b954b26b 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -146,7 +146,9 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { func TestGetGraphFromDepTree(t *testing.T) { // Create and change directory to test workspace tempDirPath, cleanUp := audit.CreateTestWorkspace(t, "gradle-example-ci-server") - defer cleanUp() + defer func() { + cleanUp() + }() assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) testCase := struct { name string From 6bb387f361cade6559bb497722093a04b2b2eaac Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 11:43:46 +0300 Subject: [PATCH 43/48] More CR Changes --- xray/audit/java/gradle.go | 45 ++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 4e0732b14..f4b212b8d 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -3,6 +3,7 @@ package java import ( "encoding/base64" "encoding/json" + "errors" "fmt" "github.com/jfrog/build-info-go/build" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -124,14 +125,12 @@ func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, de func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err error) { // Create the script file in the repository - if err = dtp.createDepTreeScript(); err != nil { + depTreeDir, err := dtp.createDepTreeScript() + if err != nil { return } defer func() { - e := os.Remove(depTreeInitFile) - if err == nil { - err = e - } + err = errors.Join(err, fileutils.RemoveTempDir(depTreeDir)) }() if dtp.useWrapper { @@ -141,10 +140,14 @@ func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err err } } - return dtp.execGradleDepTree() + return dtp.execGradleDepTree(depTreeDir) } -func (dtp *depTreeManager) createDepTreeScript() (err error) { +func (dtp *depTreeManager) createDepTreeScript() (tmpDir string, err error) { + tmpDir, err = fileutils.CreateTempDir() + if err != nil { + return + } depsRepo := "" releasesRepo := "" if dtp.server != nil { @@ -158,32 +161,34 @@ func (dtp *depTreeManager) createDepTreeScript() (err error) { } } depTreeInitScript := fmt.Sprintf(depTreeInitScript, releasesRepo, depsRepo) - return os.WriteFile(depTreeInitFile, []byte(depTreeInitScript), 0666) + return tmpDir, errorutils.CheckError(os.WriteFile(filepath.Join(tmpDir, depTreeInitFile), []byte(depTreeInitScript), 0666)) } -func (dtp *depTreeManager) execGradleDepTree() (outputFileContent []byte, err error) { +func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileContent []byte, err error) { gradleExecPath, err := build.GetGradleExecPath(dtp.useWrapper) if err != nil { + err = errorutils.CheckError(err) return } - outputFileAbsolutePath, err := filepath.Abs(depTreeOutputFile) - if err != nil { - return nil, err - } - tasks := []string{"clean", "generateDepTrees", "-I", depTreeInitFile, "-q", fmt.Sprintf("-Dcom.jfrog.depsTreeOutputFile=%s", outputFileAbsolutePath), "-Dcom.jfrog.includeAllBuildFiles=true"} + outputFilePath := filepath.Join(depTreeDir, depTreeOutputFile) + tasks := []string{ + "clean", + "generateDepTrees", "-I", filepath.Join(depTreeDir, depTreeInitFile), + "-q", + fmt.Sprintf("-Dcom.jfrog.depsTreeOutputFile=%s", outputFilePath), + "-Dcom.jfrog.includeAllBuildFiles=true"} log.Info("Running gradle dep tree command: ", gradleExecPath, tasks) if output, err := exec.Command(gradleExecPath, tasks...).CombinedOutput(); err != nil { - return nil, fmt.Errorf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output)) + return nil, errorutils.CheckErrorf(fmt.Sprintf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output))) } defer func() { - e := os.Remove(outputFileAbsolutePath) - if err == nil { - err = e - } + err = errors.Join(err, errorutils.CheckError(os.Remove(outputFilePath))) }() - return os.ReadFile(depTreeOutputFile) + outputFileContent, err = os.ReadFile(depTreeOutputFile) + err = errorutils.CheckError(err) + return } // Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input From 53393b90c1960a9a9fc41cfeb26e0cad86dfb3be Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 11:51:12 +0300 Subject: [PATCH 44/48] fix tests --- xray/audit/java/gradle.go | 2 +- xray/audit/java/gradle_test.go | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index f4b212b8d..4bedc7a34 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -186,7 +186,7 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte err = errors.Join(err, errorutils.CheckError(os.Remove(outputFilePath))) }() - outputFileContent, err = os.ReadFile(depTreeOutputFile) + outputFileContent, err = os.ReadFile(outputFilePath) err = errorutils.CheckError(err) return } diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 6b954b26b..bc7215bfb 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -201,11 +201,12 @@ func TestCreateDepTreeScript(t *testing.T) { assert.NoError(t, os.Chdir(currDir)) }() manager := &depTreeManager{} - assert.NoError(t, manager.createDepTreeScript()) + tmpDir, err = manager.createDepTreeScript() + assert.NoError(t, err) defer func() { - assert.NoError(t, os.Remove(depTreeInitFile)) + assert.NoError(t, os.Remove(filepath.Join(tmpDir, depTreeInitFile))) }() - content, err := os.ReadFile(depTreeInitFile) + content, err := os.ReadFile(filepath.Join(tmpDir, depTreeInitFile)) assert.NoError(t, err) assert.Equal(t, fmt.Sprintf(depTreeInitScript, "", ""), string(content)) manager.depsRepo = "deps-repo" @@ -214,7 +215,8 @@ func TestCreateDepTreeScript(t *testing.T) { ArtifactoryUrl: "https://myartifactory.com/artifactory", AccessToken: "my-access-token", } - assert.NoError(t, manager.createDepTreeScript()) + tmpDir, err = manager.createDepTreeScript() + assert.NoError(t, err) expectedInitScript := `initscript { repositories { maven { @@ -243,7 +245,7 @@ allprojects { } apply plugin: com.jfrog.GradleDepTree }` - content, err = os.ReadFile(depTreeInitFile) + content, err = os.ReadFile(filepath.Join(tmpDir, depTreeInitFile)) assert.NoError(t, err) assert.Equal(t, expectedInitScript, string(content)) } From 40e151eeb586328d64a75c27829a09a762944b09 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 11:54:24 +0300 Subject: [PATCH 45/48] fix tests --- go.mod | 2 +- go.sum | 4 ++-- xray/audit/java/gradle.go | 11 ++++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 648f182f3..960db5c71 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230404053100-99c7a5a74858 +replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e diff --git a/go.sum b/go.sum index f218f340a..400c8a60a 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/omerzi/jfrog-client-go v1.13.2-0.20230404053100-99c7a5a74858 h1:X40kRpPYRSihLuJ9F41ZpF8XTIc/N7xlWI/HRsZNsUw= -github.com/omerzi/jfrog-client-go v1.13.2-0.20230404053100-99c7a5a74858/go.mod h1:sBY4Z1nUXmacN+rwFkzALfS11fsZ/14nmxL7gbjULhQ= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862 h1:lQb9V8FxJ5YUvufcvdzkcNV4jWIHh3vo6xFvWFGwqpc= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862/go.mod h1:XJhlPfi6iayIVc2SQ/RbztDQOnbnNatsUSQr7wbJ8Ag= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.3 h1:1guchw824yg1CwjredY8pnzcE0SG+sfNzFY5CUYWgE4= github.com/owenrumney/go-sarif/v2 v2.1.3/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 4bedc7a34..e3931cbd1 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -3,7 +3,6 @@ package java import ( "encoding/base64" "encoding/json" - "errors" "fmt" "github.com/jfrog/build-info-go/build" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -130,7 +129,10 @@ func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err err return } defer func() { - err = errors.Join(err, fileutils.RemoveTempDir(depTreeDir)) + e := fileutils.RemoveTempDir(depTreeDir) + if err == nil { + err = e + } }() if dtp.useWrapper { @@ -183,7 +185,10 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte return nil, errorutils.CheckErrorf(fmt.Sprintf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output))) } defer func() { - err = errors.Join(err, errorutils.CheckError(os.Remove(outputFilePath))) + e := errorutils.CheckError(os.Remove(outputFilePath)) + if err == nil { + err = e + } }() outputFileContent, err = os.ReadFile(outputFilePath) From 581f4af6bd82c3bd710b2a844abbe602c54b2717 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 12:58:06 +0300 Subject: [PATCH 46/48] fix tests --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 652816488..7d5b01b82 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230404093618-e0aec23ce1c3 +replace github.com/jfrog/jfrog-client-go => github.com/omerzi/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e From ce76a2622e41582f8455d66a1a5ef7b14670bcfa Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Tue, 4 Apr 2023 12:58:31 +0300 Subject: [PATCH 47/48] fix tests --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 244d5b363..400c8a60a 100644 --- a/go.sum +++ b/go.sum @@ -202,8 +202,6 @@ github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e h1:MB5u0Kbq9 github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e/go.mod h1:HIrpwf4p4XHpAx+N+rb8SX9yrWYWs7X4rT/s0GOJfW8= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230404093618-e0aec23ce1c3 h1:I2cxiZvfEF5Gc5sL3nPsV+04ONjLVVG7v4nUn6HAVeU= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230404093618-e0aec23ce1c3/go.mod h1:XJhlPfi6iayIVc2SQ/RbztDQOnbnNatsUSQr7wbJ8Ag= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -256,6 +254,8 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862 h1:lQb9V8FxJ5YUvufcvdzkcNV4jWIHh3vo6xFvWFGwqpc= +github.com/omerzi/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862/go.mod h1:XJhlPfi6iayIVc2SQ/RbztDQOnbnNatsUSQr7wbJ8Ag= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.1.3 h1:1guchw824yg1CwjredY8pnzcE0SG+sfNzFY5CUYWgE4= github.com/owenrumney/go-sarif/v2 v2.1.3/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= From 37198293f28672ba7a00b30c0d3abd39fc8f780d Mon Sep 17 00:00:00 2001 From: Omer Zidkoni Date: Wed, 5 Apr 2023 09:26:21 +0300 Subject: [PATCH 48/48] CR Changes --- xray/audit/java/gradle.go | 2 +- xray/commands/audit/generic/auditmanager.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index e3931cbd1..145cebbe4 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -182,7 +182,7 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte "-Dcom.jfrog.includeAllBuildFiles=true"} log.Info("Running gradle dep tree command: ", gradleExecPath, tasks) if output, err := exec.Command(gradleExecPath, tasks...).CombinedOutput(); err != nil { - return nil, errorutils.CheckErrorf(fmt.Sprintf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output))) + return nil, errorutils.CheckErrorf("error running gradle-dep-tree: %s\n%s", err.Error(), string(output)) } defer func() { e := errorutils.CheckError(os.Remove(outputFilePath)) diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 9878dd233..183a7bd99 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -1,7 +1,6 @@ package audit import ( - "errors" "fmt" "github.com/jfrog/jfrog-cli-core/v2/xray/audit/java" "github.com/jfrog/jfrog-client-go/auth" @@ -215,7 +214,7 @@ func auditMultipleWorkingDirs(params *Params) (results []services.ScanResponse, } if errorList.Len() > 0 { - err = errorutils.CheckError(errors.New(errorList.String())) + err = errorutils.CheckErrorf(errorList.String()) } return @@ -279,7 +278,7 @@ func getTechDependencyTree(params *Params, tech coreutils.Technology) (dependenc case coreutils.Nuget: dependencyTrees, err = nuget.BuildDependencyTree() default: - err = errorutils.CheckError(fmt.Errorf("%s is currently not supported", string(tech))) + err = errorutils.CheckErrorf("%s is currently not supported", string(tech)) return } // Save the full dependencyTree to build impact paths for vulnerable dependencies