diff --git a/go.mod b/go.mod index d8ade8b0..c4d41ad3 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,10 @@ module github.com/orange-cloudfoundry/artifactory-resource go 1.22.6 - require ( github.com/Masterminds/semver v1.5.0 - github.com/jfrog/jfrog-cli-core/v2 v2.55.7 - github.com/jfrog/jfrog-client-go v1.46.2 + github.com/jfrog/jfrog-cli-core/v2 v2.56.0 + github.com/jfrog/jfrog-client-go v1.47.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -38,7 +37,7 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jedib0t/go-pretty/v6 v6.5.9 // indirect github.com/jfrog/archiver/v3 v3.6.1 // indirect - github.com/jfrog/build-info-go v1.9.36 // indirect + github.com/jfrog/build-info-go v1.10.0 // indirect github.com/jfrog/gofrog v1.7.6 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.9 // indirect @@ -75,8 +74,8 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.27.0 // indirect - golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect diff --git a/go.sum b/go.sum index 55b851de..bdcee18f 100644 --- a/go.sum +++ b/go.sum @@ -92,14 +92,14 @@ github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+ github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jfrog/archiver/v3 v3.6.1 h1:LOxnkw9pOn45DzCbZNFV6K0+6dCsQ0L8mR3ZcujO5eI= github.com/jfrog/archiver/v3 v3.6.1/go.mod h1:VgR+3WZS4N+i9FaDwLZbq+jeU4B4zctXL+gL4EMzfLw= -github.com/jfrog/build-info-go v1.9.36 h1:bKoYW3o+U70Zbz2kt5NT84N5JWNxdDXHOf+kVdzK+j4= -github.com/jfrog/build-info-go v1.9.36/go.mod h1:JcISnovFXKx3wWf3p1fcMmlPdt6adxScXvoJN4WXqIE= +github.com/jfrog/build-info-go v1.10.0 h1:jSxmN58mH0LaP+v1IQadplwJPRILLgI3xieBTXTCSos= +github.com/jfrog/build-info-go v1.10.0/go.mod h1:JcISnovFXKx3wWf3p1fcMmlPdt6adxScXvoJN4WXqIE= github.com/jfrog/gofrog v1.7.6 h1:QmfAiRzVyaI7JYGsB7cxfAJePAZTzFz0gRWZSE27c6s= github.com/jfrog/gofrog v1.7.6/go.mod h1:ntr1txqNOZtHplmaNd7rS4f8jpA5Apx8em70oYEe7+4= -github.com/jfrog/jfrog-cli-core/v2 v2.55.7 h1:V4dO2FMNIH49lov3dMj3jYRg8KBTG7hyhHI8ftYByf8= -github.com/jfrog/jfrog-cli-core/v2 v2.55.7/go.mod h1:DPO5BfWAeOByahFMMy+PcjmbPlcyoRy7Bf2C5sGKVi0= -github.com/jfrog/jfrog-client-go v1.46.2 h1:1rk7PliYGc7zVSFVE2/RO77JOR1KdEtr28os8GQiLyI= -github.com/jfrog/jfrog-client-go v1.46.2/go.mod h1:qtQ9ML8xrRJmUwU/t6QRsov7C5mIZndTDY3qulgB5hA= +github.com/jfrog/jfrog-cli-core/v2 v2.56.0 h1:rCNKhfESgsq0o6//gU1mNCvuCboE5BMfycj/RM/gq8k= +github.com/jfrog/jfrog-cli-core/v2 v2.56.0/go.mod h1:D8m0L8GCZiYCY9MjhnWY4egCqyVlU2iZsVA0yysBsVw= +github.com/jfrog/jfrog-client-go v1.47.0 h1:OBMB6TxqziBByjuk6hm0BM30pQwOb3XzjZKf/cmwCeM= +github.com/jfrog/jfrog-client-go v1.47.0/go.mod h1:UxzL9Q4pDoM+HQjSuQiGNakyoJNuxqPSs35/amBJvdY= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -227,8 +227,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= -golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -238,8 +238,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vendor/github.com/jfrog/build-info-go/build/build.go b/vendor/github.com/jfrog/build-info-go/build/build.go index 10b3aaef..72d872d3 100644 --- a/vendor/github.com/jfrog/build-info-go/build/build.go +++ b/vendor/github.com/jfrog/build-info-go/build/build.go @@ -373,6 +373,14 @@ func (b *Build) GetBuildTimestamp() time.Time { return b.buildTimestamp } +func (b *Build) AddArtifacts(moduleId string, moduleType entities.ModuleType, artifacts ...entities.Artifact) error { + if !b.buildNameAndNumberProvided() { + return errors.New("a build name must be provided in order to add artifacts") + } + partial := &entities.Partial{ModuleId: moduleId, ModuleType: moduleType, Artifacts: artifacts} + return b.SavePartialBuildInfo(partial) +} + type partialModule struct { moduleType entities.ModuleType artifacts map[string]entities.Artifact diff --git a/vendor/github.com/jfrog/build-info-go/build/golang.go b/vendor/github.com/jfrog/build-info-go/build/golang.go index 1ccbe207..5dbcf5df 100644 --- a/vendor/github.com/jfrog/build-info-go/build/golang.go +++ b/vendor/github.com/jfrog/build-info-go/build/golang.go @@ -55,11 +55,7 @@ func (gm *GoModule) SetName(name string) { } func (gm *GoModule) AddArtifacts(artifacts ...entities.Artifact) error { - if !gm.containingBuild.buildNameAndNumberProvided() { - return errors.New("a build name must be provided in order to add artifacts") - } - partial := &entities.Partial{ModuleId: gm.name, ModuleType: entities.Go, Artifacts: artifacts} - return gm.containingBuild.SavePartialBuildInfo(partial) + return gm.containingBuild.AddArtifacts(gm.name, entities.Go, artifacts...) } func (gm *GoModule) loadDependencies() ([]entities.Dependency, error) { diff --git a/vendor/github.com/jfrog/build-info-go/build/npm.go b/vendor/github.com/jfrog/build-info-go/build/npm.go index 6fe77db7..d5b7d0bb 100644 --- a/vendor/github.com/jfrog/build-info-go/build/npm.go +++ b/vendor/github.com/jfrog/build-info-go/build/npm.go @@ -97,11 +97,7 @@ func (nm *NpmModule) SetCollectBuildInfo(collectBuildInfo bool) { } func (nm *NpmModule) AddArtifacts(artifacts ...entities.Artifact) error { - if !nm.containingBuild.buildNameAndNumberProvided() { - return errors.New("a build name must be provided in order to add artifacts") - } - partial := &entities.Partial{ModuleId: nm.name, ModuleType: entities.Npm, Artifacts: artifacts} - return nm.containingBuild.SavePartialBuildInfo(partial) + return nm.containingBuild.AddArtifacts(nm.name, entities.Npm, artifacts...) } // This function discards the npm command in npmArgs and keeps only the command flags. diff --git a/vendor/github.com/jfrog/build-info-go/build/python.go b/vendor/github.com/jfrog/build-info-go/build/python.go index 60299dc2..ee036c01 100644 --- a/vendor/github.com/jfrog/build-info-go/build/python.go +++ b/vendor/github.com/jfrog/build-info-go/build/python.go @@ -11,7 +11,7 @@ import ( type PythonModule struct { containingBuild *Build tool pythonutils.PythonTool - name string + id string srcPath string localDependenciesPath string updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error @@ -37,33 +37,40 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e if err != nil { return fmt.Errorf("failed while attempting to get %s dependencies graph: %s", pm.tool, err.Error()) } - // Get package-name. - packageName, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath) - if pkgNameErr != nil { - pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error()) - } - // If module-name was set by the command, don't change it. - if pm.name == "" { - // If the package name is unknown, set the module name to be the build name. - pm.name = packageName - if pm.name == "" { - pm.name = pm.containingBuild.buildName - pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.name)) - } - } + + packageId := pm.SetModuleId() + if pm.updateDepsChecksumInfoFunc != nil { err = pm.updateDepsChecksumInfoFunc(dependenciesMap, pm.srcPath) if err != nil { return err } } - pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageName, pm.name) - buildInfoModule := entities.Module{Id: pm.name, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)} + pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageId, pm.id) + buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)} buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}} return pm.containingBuild.SaveBuildInfo(buildInfo) } +// Sets the module ID and returns the package ID (if found). +func (pm *PythonModule) SetModuleId() (packageId string) { + packageId, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath) + if pkgNameErr != nil { + pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error()) + } + // If module-name was set by the command, don't change it. + if pm.id == "" { + // If the package name is unknown, set the module name to be the build name. + pm.id = packageId + if pm.id == "" { + pm.id = pm.containingBuild.buildName + pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.id)) + } + } + return +} + // Run install command while parsing the logs for downloaded packages. // Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...). func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) { @@ -71,7 +78,7 @@ func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string] } func (pm *PythonModule) SetName(name string) { - pm.name = name + pm.id = name } func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) { @@ -81,3 +88,27 @@ func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) { func (pm *PythonModule) SetUpdateDepsChecksumInfoFunc(updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error) { pm.updateDepsChecksumInfoFunc = updateDepsChecksumInfoFunc } + +func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]entities.Artifact, error) { + pm.SetModuleId() + artifactsPaths, err := pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath) + if err != nil { + return nil, err + } + return pythonutils.CreateArtifactsFromPaths(artifactsPaths) +} + +func (pm *PythonModule) AddArtifacts(artifacts []entities.Artifact) error { + return pm.containingBuild.AddArtifacts(pm.id, entities.Python, artifacts...) +} + +func (pm *PythonModule) TwineUploadAndGenerateBuild(commandArgs []string) error { + artifacts, err := pm.TwineUploadWithLogParsing(commandArgs) + if err != nil { + return err + } + + buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Artifacts: artifacts} + buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}} + return pm.containingBuild.SaveBuildInfo(buildInfo) +} diff --git a/vendor/github.com/jfrog/build-info-go/build/yarn.go b/vendor/github.com/jfrog/build-info-go/build/yarn.go index fcd436ac..bf3bd1be 100644 --- a/vendor/github.com/jfrog/build-info-go/build/yarn.go +++ b/vendor/github.com/jfrog/build-info-go/build/yarn.go @@ -149,11 +149,7 @@ func (ym *YarnModule) SetTraverseDependenciesFunc(traverseDependenciesFunc func( } func (ym *YarnModule) AddArtifacts(artifacts ...entities.Artifact) error { - if !ym.containingBuild.buildNameAndNumberProvided() { - return errors.New("a build name must be provided in order to add artifacts") - } - partial := &entities.Partial{ModuleId: ym.name, ModuleType: entities.Npm, Artifacts: artifacts} - return ym.containingBuild.SavePartialBuildInfo(partial) + return ym.containingBuild.AddArtifacts(ym.name, entities.Npm, artifacts...) } func validateYarnVersion(executablePath, srcPath string) error { diff --git a/vendor/github.com/jfrog/build-info-go/entities/buildinfo.go b/vendor/github.com/jfrog/build-info-go/entities/buildinfo.go index 6221dde3..f330ac54 100644 --- a/vendor/github.com/jfrog/build-info-go/entities/buildinfo.go +++ b/vendor/github.com/jfrog/build-info-go/entities/buildinfo.go @@ -6,6 +6,7 @@ import ( "golang.org/x/exp/maps" "golang.org/x/exp/slices" "regexp" + "sort" "strings" "time" @@ -200,13 +201,24 @@ func (targetBuildInfo *BuildInfo) ToCycloneDxBom() (*cdx.BOM, error) { } } + sort.Slice(components, func(i, j int) bool { + return components[i].BOMRef < components[j].BOMRef + }) + // Convert the map of dependencies to CycloneDX dependency objects var dependencies []cdx.Dependency for compRef, deps := range depMap { depsSlice := maps.Keys(deps) + sort.Slice(depsSlice, func(i, j int) bool { + return depsSlice[i] < depsSlice[j] + }) dependencies = append(dependencies, cdx.Dependency{Ref: compRef, Dependencies: &depsSlice}) } + sort.Slice(dependencies, func(i, j int) bool { + return dependencies[i].Ref < dependencies[j].Ref + }) + bom := cdx.NewBOM() bom.Components = &components bom.Dependencies = &dependencies diff --git a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/piputils.go b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/piputils.go index b420aa19..2df46593 100644 --- a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/piputils.go +++ b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/piputils.go @@ -73,20 +73,20 @@ func writeScriptIfNeeded(targetDirPath, scriptName string) error { return nil } -func getPackageNameFromSetuppy(srcPath string) (string, error) { +func getPackageDetailsFromSetuppy(srcPath string) (packageName string, packageVersion string, err error) { filePath, err := getSetupPyFilePath(srcPath) if err != nil || filePath == "" { // Error was returned or setup.py does not exist in directory. - return "", err + return } // Extract package name from setup.py. - packageName, err := ExtractPackageNameFromSetupPy(filePath) + packageName, packageVersion, err = extractPackageNameFromSetupPy(filePath) if err != nil { // If setup.py egg_info command failed we use build name as module name and continue to pip-install execution - return "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error()) + return "", "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error()) } - return packageName, nil + return packageName, packageVersion, nil } // Look for 'setup.py' file in current work dir. @@ -95,16 +95,16 @@ func getSetupPyFilePath(srcPath string) (string, error) { return getFilePath(srcPath, "setup.py") } -// Get the project-name by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file. -func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, error) { +// Get the project name and version by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file. +func extractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) { // Execute egg_info command and return PKG-INFO content. content, err := getEgginfoPkginfoContent(setuppyFilePath) if err != nil { - return "", err + return "", "", err } // Extract project name from file content. - return getProjectIdFromFileContent(content) + return getProjectNameAndVersionFromFileContent(content) } // Run egg-info command on setup.py. The command generates metadata files. @@ -182,24 +182,51 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) { // Get package ID from PKG-INFO file content. // If pattern of package name of version not found, return an error. -func getProjectIdFromFileContent(content []byte) (string, error) { +func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) { // Create package-name regexp. - packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s(\w[\w-.]+)`) + packageNameWithPrefixRegexp := regexp.MustCompile(`(?m)^Name:\s` + packageNameRegexp) // Find first nameMatch of packageNameRegexp. - nameMatch := packageNameRegexp.FindStringSubmatch(string(content)) + nameMatch := packageNameWithPrefixRegexp.FindStringSubmatch(string(content)) if len(nameMatch) < 2 { - return "", errors.New("failed extracting package name from content") + return "", "", errors.New("failed extracting package name from content") } // Create package-version regexp. - packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s(\w[\w-.]+)`) + packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + packageNameRegexp) // Find first match of packageNameRegexp. versionMatch := packageVersionRegexp.FindStringSubmatch(string(content)) if len(versionMatch) < 2 { - return "", errors.New("failed extracting package version from content") + return "", "", errors.New("failed extracting package version from content") } - return nameMatch[1] + ":" + versionMatch[1], nil + return nameMatch[1], versionMatch[1], nil +} + +// Try getting the name and version from pyproject.toml or from setup.py, if those exist. +func getPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) { + projectName, projectVersion, err = getPipProjectDetailsFromPyProjectToml(srcPath) + if err != nil || projectName != "" { + return + } + return getPackageDetailsFromSetuppy(srcPath) +} + +// Returns project ID based on name and version from pyproject.toml or setup.py, if found. +func getPipProjectId(srcPath string) (string, error) { + projectName, projectVersion, err := getPipProjectNameAndVersion(srcPath) + if err != nil || projectName == "" { + return "", err + } + return projectName + ":" + projectVersion, nil +} + +// Try getting the name and version from pyproject.toml. +func getPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) { + filePath, err := getPyProjectFilePath(srcPath) + if err != nil || filePath == "" { + return + } + return extractPipProjectDetailsFromPyProjectToml(filePath) } diff --git a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/poetryutils.go b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/poetryutils.go index 03fe56f8..69b7878a 100644 --- a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/poetryutils.go +++ b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/poetryutils.go @@ -9,9 +9,6 @@ import ( "golang.org/x/exp/maps" ) -type PyprojectToml struct { - Tool map[string]PoetryPackage -} type PoetryPackage struct { Name string Version string @@ -31,7 +28,7 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep // Error was returned or poetry.lock does not exist in directory. return map[string][]string{}, []string{}, err } - projectName, directDependencies, err := getPackageNameFromPyproject(srcPath) + projectName, directDependencies, err := getPoetryPackageFromPyProject(srcPath) if err != nil { return map[string][]string{}, []string{}, err } @@ -56,49 +53,36 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep return graph, graph[projectName], nil } -func getPackageNameFromPyproject(srcPath string) (string, []string, error) { - filePath, err := getPyprojectFilePath(srcPath) +func getPoetryPackageFromPyProject(srcPath string) (string, []string, error) { + filePath, err := getPyProjectFilePath(srcPath) if err != nil || filePath == "" { - // Error was returned or pyproject.toml does not exist in directory. return "", []string{}, err } - // Extract package name from pyproject.toml. - project, err := extractProjectFromPyproject(filePath) + project, err := extractPoetryPackageFromPyProjectToml(filePath) if err != nil { return "", []string{}, err } return project.Name, append(maps.Keys(project.Dependencies), maps.Keys(project.DevDependencies)...), nil } -// Look for 'pyproject.toml' file in current work dir. -// If found, return its absolute path. -func getPyprojectFilePath(srcPath string) (string, error) { - return getFilePath(srcPath, "pyproject.toml") -} - // Look for 'poetry.lock' file in current work dir. // If found, return its absolute path. func getPoetryLockFilePath(srcPath string) (string, error) { return getFilePath(srcPath, "poetry.lock") } -// Get the project-name by parsing the pyproject.toml file. -func extractProjectFromPyproject(pyprojectFilePath string) (project PoetryPackage, err error) { - content, err := os.ReadFile(pyprojectFilePath) - if err != nil { - return - } - var pyprojectFile PyprojectToml - _, err = toml.Decode(string(content), &pyprojectFile) +// Get poetry package by parsing the pyproject.toml file. +func extractPoetryPackageFromPyProjectToml(pyProjectFilePath string) (project PoetryPackage, err error) { + pyProjectFile, err := decodePyProjectToml(pyProjectFilePath) if err != nil { return } - if poetryProject, ok := pyprojectFile.Tool["poetry"]; ok { + if poetryProject, ok := pyProjectFile.Tool["poetry"]; ok { // Extract project name from file content. poetryProject.Name = poetryProject.Name + ":" + poetryProject.Version return poetryProject, nil } - return PoetryPackage{}, errors.New("Couldn't find project name and version in " + pyprojectFilePath) + return PoetryPackage{}, errors.New("Couldn't find project name and version in " + pyProjectFilePath) } // Get the project-name by parsing the poetry.lock file diff --git a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/pyprojectutils.go b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/pyprojectutils.go new file mode 100644 index 00000000..81c9f67a --- /dev/null +++ b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/pyprojectutils.go @@ -0,0 +1,45 @@ +package pythonutils + +import ( + "github.com/BurntSushi/toml" + "os" +) + +type PyProjectToml struct { + // Represents the [tool.poetry] section in pyproject.toml. + Tool map[string]PoetryPackage + // Represents the [project] section in pyproject.toml, for pypi package managers other than poetry. + Project Project +} + +// Pypi project defined for package managers other than poetry (pip, pipenv, etc...) +type Project struct { + Name string + Version string + Description string +} + +// Get project name and version by parsing the pyproject.toml file. +// Try to extract the project name and version from the [project] section, or if requested also try from [tool.poetry] section. +func extractPipProjectDetailsFromPyProjectToml(pyProjectFilePath string) (projectName, projectVersion string, err error) { + pyProjectFile, err := decodePyProjectToml(pyProjectFilePath) + if err != nil { + return + } + return pyProjectFile.Project.Name, pyProjectFile.Project.Version, nil +} + +func decodePyProjectToml(pyProjectFilePath string) (pyProjectFile PyProjectToml, err error) { + content, err := os.ReadFile(pyProjectFilePath) + if err != nil { + return + } + _, err = toml.Decode(string(content), &pyProjectFile) + return +} + +// Look for 'pyproject.toml' file in current work dir. +// If found, return its absolute path. +func getPyProjectFilePath(srcPath string) (string, error) { + return getFilePath(srcPath, "pyproject.toml") +} diff --git a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/twineutils.go b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/twineutils.go new file mode 100644 index 00000000..4148d4b9 --- /dev/null +++ b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/twineutils.go @@ -0,0 +1,89 @@ +package pythonutils + +import ( + "fmt" + "github.com/jfrog/build-info-go/entities" + "github.com/jfrog/gofrog/crypto" + gofrogcmd "github.com/jfrog/gofrog/io" + "github.com/jfrog/gofrog/log" + "path" + "path/filepath" + "regexp" + "slices" + "strings" +) + +const ( + _twineExeName = "twine" + _twineUploadCmdName = "upload" + _verboseFlag = "--verbose" + _disableProgressFlag = "--disable-progress-bar" +) + +// Run a twine upload and parse artifacts paths from logs. +func TwineUploadWithLogParsing(commandArgs []string, srcPath string) (artifactsPaths []string, err error) { + commandArgs = addRequiredFlags(commandArgs) + uploadCmd := gofrogcmd.NewCommand(_twineExeName, _twineUploadCmdName, commandArgs) + uploadCmd.Dir = srcPath + log.Debug("Running twine command: '", _twineExeName, _twineUploadCmdName, strings.Join(commandArgs, " "), "'with build info collection") + _, errorOut, _, err := gofrogcmd.RunCmdWithOutputParser(uploadCmd, true, getArtifactsParser(&artifactsPaths)) + if err != nil { + return nil, fmt.Errorf("failed running '%s %s %s' command with error: '%s - %s'", _twineExeName, _twineUploadCmdName, strings.Join(commandArgs, " "), err.Error(), errorOut) + } + return +} + +// Enabling verbose and disabling progress bar are required for log parsing. +func addRequiredFlags(commandArgs []string) []string { + for _, flag := range []string{_verboseFlag, _disableProgressFlag} { + if !slices.Contains(commandArgs, flag) { + commandArgs = append(commandArgs, flag) + } + } + return commandArgs +} + +func getArtifactsParser(artifactsPaths *[]string) (parser *gofrogcmd.CmdOutputPattern) { + return &gofrogcmd.CmdOutputPattern{ + // Regexp to catch the paths in lines such as "INFO dist/jfrog_python_example-1.0-py3-none-any.whl (1.6 KB)" + // First part ".+\s" is the line prefix. + // Second part "([^ \t]+)" is the artifact path as a group. + // Third part "\s+\([\d.]+\s+[A-Za-z]{2}\)" is the size and unit, surrounded by parentheses. + RegExp: regexp.MustCompile(`^.+\s([^ \t]+)\s+\([\d.]+\s+[A-Za-z]{2}\)`), + ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + // Check for out of bound results. + if len(pattern.MatchedResults)-1 <= 0 { + log.Debug(fmt.Sprintf("Failed extracting artifact name from line: %s", pattern.Line)) + return pattern.Line, nil + } + *artifactsPaths = append(*artifactsPaths, pattern.MatchedResults[1]) + return pattern.Line, nil + }, + } +} + +// Create artifacts entities from the artifacts paths that were found during the upload. +func CreateArtifactsFromPaths(artifactsPaths []string) (artifacts []entities.Artifact, err error) { + projectName, projectVersion, err := getPipProjectNameAndVersion("") + if err != nil { + return + } + var absPath string + var fileDetails *crypto.FileDetails + for _, artifactPath := range artifactsPaths { + absPath, err = filepath.Abs(artifactPath) + if err != nil { + return nil, err + } + fileDetails, err = crypto.GetFileDetails(absPath, true) + if err != nil { + return nil, err + } + + artifact := entities.Artifact{Name: filepath.Base(absPath), Path: path.Join(projectName, projectVersion, filepath.Base(absPath)), + Type: strings.TrimPrefix(filepath.Ext(absPath), ".")} + artifact.Checksum = entities.Checksum{Sha1: fileDetails.Checksum.Sha1, Md5: fileDetails.Checksum.Md5} + artifacts = append(artifacts, artifact) + } + return +} diff --git a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/utils.go b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/utils.go index 483b5e03..9db3657b 100644 --- a/vendor/github.com/jfrog/build-info-go/utils/pythonutils/utils.go +++ b/vendor/github.com/jfrog/build-info-go/utils/pythonutils/utils.go @@ -18,12 +18,14 @@ const ( Pip PythonTool = "pip" Pipenv PythonTool = "pipenv" Poetry PythonTool = "poetry" + Twine PythonTool = "twine" startDownloadingPattern = `^\s*Downloading\s` downloadingCaptureGroup = `[^\s]*` startUsingCachedPattern = `^\s*Using\scached\s` usingCacheCaptureGroup = `[\S]+` endPattern = `\s\(` + packageNameRegexp = `(\w[\w-.]+)` ) type PythonTool string @@ -99,10 +101,10 @@ func GetPythonDependencies(tool PythonTool, srcPath, localDependenciesPath strin func GetPackageName(tool PythonTool, srcPath string) (packageName string, err error) { switch tool { - case Pip, Pipenv: - return getPackageNameFromSetuppy(srcPath) + case Pip, Pipenv, Twine: + return getPipProjectId(srcPath) case Poetry: - packageName, _, err = getPackageNameFromPyproject(srcPath) + packageName, _, err = getPoetryPackageFromPyProject(srcPath) return default: return "", errors.New(string(tool) + " commands are not supported.") @@ -148,7 +150,7 @@ func updateDepsIdsAndRequestedBy(parentDependency entities.Dependency, dependenc } childDep.Type = fileType } - // Convert Id field from filename to dependency id + // Convert ID field from filename to dependency id childDep.Id = childId // Reassign map entry with new entry copy dependenciesMap[childName] = childDep @@ -213,7 +215,7 @@ func getMultilineSplitCaptureOutputPattern(startCollectingPattern, captureGroup, } // Mask the pre-known credentials that are provided as command arguments from logs. -// This function creates a log parser for each credentials argument. +// This function creates a log parser for each credentials' argument. func maskPreKnownCredentials(args []string) (parsers []*gofrogcmd.CmdOutputPattern) { for _, arg := range args { // If this argument is a credentials argument, create a log parser that masks it. @@ -268,14 +270,14 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, installCmd.Dir = srcPath dependenciesMap := map[string]entities.Dependency{} - parsers := []*gofrogcmd.CmdOutputPattern{} + var parsers []*gofrogcmd.CmdOutputPattern var packageName string expectingPackageFilePath := false // Extract downloaded package name. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ - RegExp: regexp.MustCompile(`^Collecting\s(\w[\w-.]+)`), + RegExp: regexp.MustCompile(`^Collecting\s` + packageNameRegexp), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // If this pattern matched a second time before downloaded-file-name was found, prompt a message. if expectingPackageFilePath { @@ -328,7 +330,7 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, // Extract already installed packages names. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ - RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`), + RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s` + packageNameRegexp), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // Check for out of bound results. if len(pattern.MatchedResults)-1 < 0 { diff --git a/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic/upload.go b/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic/upload.go index 24f0c208..8b721ffd 100644 --- a/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic/upload.go +++ b/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic/upload.go @@ -3,6 +3,7 @@ package generic import ( "errors" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/commandsummary" + "github.com/jfrog/jfrog-client-go/artifactory" "os" buildInfo "github.com/jfrog/build-info-go/entities" @@ -135,7 +136,7 @@ func (uc *UploadCommand) upload() (err error) { var artifactsDetailsReader *content.ContentReader = nil if uc.DetailedSummary() || toCollect { var summary *rtServicesUtils.OperationSummary - summary, err = servicesManager.UploadFilesWithSummary(uploadParamsArray...) + summary, err = servicesManager.UploadFilesWithSummary(artifactory.UploadServiceOptions{}, uploadParamsArray...) if err != nil { errorOccurred = true log.Error(err) @@ -162,7 +163,7 @@ func (uc *UploadCommand) upload() (err error) { } } } else { - successCount, failCount, err = servicesManager.UploadFiles(uploadParamsArray...) + successCount, failCount, err = servicesManager.UploadFiles(artifactory.UploadServiceOptions{}, uploadParamsArray...) if err != nil { errorOccurred = true log.Error(err) diff --git a/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/commandsummary/buildinfosummary.go b/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/commandsummary/buildinfosummary.go index 2a0fe9e0..9861c723 100644 --- a/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/commandsummary/buildinfosummary.go +++ b/vendor/github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/commandsummary/buildinfosummary.go @@ -31,6 +31,7 @@ var ( buildInfo.Generic: true, buildInfo.Terraform: true, buildInfo.Docker: true, + buildInfo.Python: true, } ) diff --git a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/config/encryption.go b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/config/encryption.go index eb4db444..2c78afb7 100644 --- a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/config/encryption.go +++ b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/config/encryption.go @@ -193,7 +193,9 @@ func decrypt(encryptedSecret string, key string) (string, error) { return "", errorutils.CheckErrorf(decryptErrorPrefix + "unexpected cipher text size") } + //#nosec G407 nonce, cipherText := cipherText[:nonceSize], cipherText[nonceSize:] + //#nosec G407 plaintext, err := gcm.Open(nil, nonce, cipherText, nil) if err != nil { return "", errorutils.CheckError(err) diff --git a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/cmdutils.go b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/cmdutils.go index 4ef9fe34..d57783c7 100644 --- a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/cmdutils.go +++ b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/cmdutils.go @@ -138,7 +138,7 @@ func FindFlagFirstMatch(flags, args []string) (flagIndex, flagValueIndex int, fl } func ExtractServerIdFromCommand(args []string) (cleanArgs []string, serverId string, err error) { - return extractStringOptionFromArgs(args, "server-id") + return ExtractStringOptionFromArgs(args, "server-id") } func ExtractThreadsFromArgs(args []string, defaultValue int) (cleanArgs []string, threads int, err error) { @@ -181,12 +181,12 @@ func ExtractLicensesFromArgs(args []string) (cleanArgs []string, licenses bool, // Used by docker scan (Xray) func ExtractRepoPathFromArgs(args []string) (cleanArgs []string, repoPath string, err error) { - return extractStringOptionFromArgs(args, "repo-path") + return ExtractStringOptionFromArgs(args, "repo-path") } // Used by docker scan (Xray) func ExtractWatchesFromArgs(args []string) (cleanArgs []string, watches string, err error) { - return extractStringOptionFromArgs(args, "watches") + return ExtractStringOptionFromArgs(args, "watches") } func ExtractDetailedSummaryFromArgs(args []string) (cleanArgs []string, detailedSummary bool, err error) { @@ -198,14 +198,14 @@ func ExtractXrayScanFromArgs(args []string) (cleanArgs []string, xrayScan bool, } func ExtractXrayOutputFormatFromArgs(args []string) (cleanArgs []string, format string, err error) { - return extractStringOptionFromArgs(args, "format") + return ExtractStringOptionFromArgs(args, "format") } func ExtractTagFromArgs(args []string) (cleanArgs []string, tag string, err error) { - return extractStringOptionFromArgs(args, "tag") + return ExtractStringOptionFromArgs(args, "tag") } -func extractStringOptionFromArgs(args []string, optionName string) (cleanArgs []string, value string, err error) { +func ExtractStringOptionFromArgs(args []string, optionName string) (cleanArgs []string, value string, err error) { cleanArgs = append([]string(nil), args...) flagIndex, valIndex, value, err := FindFlag("--"+optionName, cleanArgs) diff --git a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/profiler.go b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/profiler.go index 94626e6a..b4986e52 100644 --- a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/profiler.go +++ b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/profiler.go @@ -3,6 +3,7 @@ package coreutils import ( "errors" "fmt" + "github.com/jfrog/gofrog/safeconvert" "os" "runtime/pprof" "time" @@ -70,7 +71,11 @@ func (p *Profiler) threadDumpToFile() (outputFilePath string, err error) { err = errors.Join(err, errorutils.CheckError(outputFile.Close())) }() - for i := 0; i < int(p.repetitions); i++ { + signedRepetitions, err := safeconvert.UintToInt(p.repetitions) + if err != nil { + return "", fmt.Errorf("failed to convert repetitions to int: %w", err) + } + for i := 0; i < signedRepetitions; i++ { fmt.Fprintf(outputFile, "========== Thread dump #%d ==========\n", i) prof := pprof.Lookup("goroutine") if err = errorutils.CheckError(prof.WriteTo(outputFile, 1)); err != nil { diff --git a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/tableutils.go b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/tableutils.go index cfa3a3eb..ebccc161 100644 --- a/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/tableutils.go +++ b/vendor/github.com/jfrog/jfrog-cli-core/v2/utils/coreutils/tableutils.go @@ -153,7 +153,7 @@ func PrepareTable(rows interface{}, emptyTableMessage string, printExtended bool rowsSliceValue := reflect.ValueOf(rows) if rowsSliceValue.Len() == 0 && emptyTableMessage != "" { - PrintMessage(emptyTableMessage) + PrintMessageInsideFrame(emptyTableMessage, "") return nil, nil } @@ -367,15 +367,27 @@ type embeddedTableCell struct { // PrintMessage prints message in a frame (which is actually a table with a single table). // For example: +// // ┌─────────────────────────────────────────┐ // │ An example of a message in a nice frame │ // └─────────────────────────────────────────┘ -func PrintMessage(message string) { +// With a margin left of 4 spaces: +// +// ┌─────────────────────────────────────────┐ +// │ An example of a message in a nice frame │ +// └─────────────────────────────────────────┘ +func PrintMessageInsideFrame(message, marginLeft string) { tableWriter := table.NewWriter() tableWriter.SetOutputMirror(os.Stdout) if log.IsStdOutTerminal() { tableWriter.SetStyle(table.StyleLight) } + + // Set margin left for the whole frame (for example, " "). + tableWriter.Style().Box.Left = marginLeft + tableWriter.Style().Box.Left + tableWriter.Style().Box.TopLeft = marginLeft + tableWriter.Style().Box.TopLeft + tableWriter.Style().Box.BottomLeft = marginLeft + tableWriter.Style().Box.BottomLeft + // Remove emojis from non-supported terminals message = RemoveEmojisIfNonSupportedTerminal(message) tableWriter.AppendRow(table.Row{message}) diff --git a/vendor/github.com/jfrog/jfrog-client-go/artifactory/emptymanager.go b/vendor/github.com/jfrog/jfrog-client-go/artifactory/emptymanager.go index 2060c10f..a7149ca1 100644 --- a/vendor/github.com/jfrog/jfrog-client-go/artifactory/emptymanager.go +++ b/vendor/github.com/jfrog/jfrog-client-go/artifactory/emptymanager.go @@ -56,9 +56,8 @@ type ArtifactoryServicesManager interface { SetProps(params services.PropsParams) (int, error) DeleteProps(params services.PropsParams) (int, error) GetItemProps(relativePath string) (*utils.ItemProperties, error) - UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) - UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) - UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) + UploadFiles(uploadServiceOptions UploadServiceOptions, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) + UploadFilesWithSummary(uploadServiceOptions UploadServiceOptions, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) Move(params ...services.MoveCopyParams) (successCount, failedCount int, err error) PublishGoProject(params _go.GoParams) (*utils.OperationSummary, error) @@ -263,15 +262,11 @@ func (esm *EmptyArtifactoryServicesManager) GetItemProps(string) (*utils.ItemPro panic("Failed: Method is not implemented") } -func (esm *EmptyArtifactoryServicesManager) UploadFiles(...services.UploadParams) (int, int, error) { +func (esm *EmptyArtifactoryServicesManager) UploadFiles(_ UploadServiceOptions, _ ...services.UploadParams) (int, int, error) { panic("Failed: Method is not implemented") } -func (esm *EmptyArtifactoryServicesManager) UploadFilesWithFailFast(...services.UploadParams) (int, int, error) { - panic("Failed: Method is not implemented") -} - -func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(...services.UploadParams) (*utils.OperationSummary, error) { +func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(_ UploadServiceOptions, _ ...services.UploadParams) (*utils.OperationSummary, error) { panic("Failed: Method is not implemented") } diff --git a/vendor/github.com/jfrog/jfrog-client-go/artifactory/manager.go b/vendor/github.com/jfrog/jfrog-client-go/artifactory/manager.go index 10a785bf..f79409e8 100644 --- a/vendor/github.com/jfrog/jfrog-client-go/artifactory/manager.go +++ b/vendor/github.com/jfrog/jfrog-client-go/artifactory/manager.go @@ -302,37 +302,34 @@ func (sm *ArtifactoryServicesManagerImp) GetItemProps(relativePath string) (*uti return setPropsService.GetItemProperties(relativePath) } -func (sm *ArtifactoryServicesManagerImp) initUploadService() *services.UploadService { +type UploadServiceOptions struct { + // Fail the operation immediately if an error occurs. + failFast bool +} + +func (sm *ArtifactoryServicesManagerImp) initUploadService(uploadServiceOptions UploadServiceOptions) *services.UploadService { uploadService := services.NewUploadService(sm.client) uploadService.Threads = sm.config.GetThreads() uploadService.ArtDetails = sm.config.GetServiceDetails() uploadService.DryRun = sm.config.IsDryRun() + uploadService.SetFailFast(uploadServiceOptions.failFast) uploadService.Progress = sm.progress httpClientDetails := uploadService.ArtDetails.CreateHttpClientDetails() uploadService.MultipartUpload = utils.NewMultipartUpload(sm.client, &httpClientDetails, uploadService.ArtDetails.GetUrl()) return uploadService } -func (sm *ArtifactoryServicesManagerImp) UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { - return sm.uploadFiles(sm.initUploadService(), params...) -} - -func (sm *ArtifactoryServicesManagerImp) UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { - uploadService := sm.initUploadService() - uploadService.SetFailFast(true) - return sm.uploadFiles(uploadService, params...) -} - -func (sm *ArtifactoryServicesManagerImp) uploadFiles(uploadService *services.UploadService, uploadParams ...services.UploadParams) (totalUploaded, totalFailed int, err error) { - summary, err := uploadService.UploadFiles(uploadParams...) +func (sm *ArtifactoryServicesManagerImp) UploadFiles(uploadServiceOptions UploadServiceOptions, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { + uploadService := sm.initUploadService(uploadServiceOptions) + summary, err := uploadService.UploadFiles(params...) if summary == nil { return 0, 0, err } return summary.TotalSucceeded, summary.TotalFailed, err } -func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) { - uploadService := sm.initUploadService() +func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(uploadServiceOptions UploadServiceOptions, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) { + uploadService := sm.initUploadService(uploadServiceOptions) uploadService.SetSaveSummary(true) return uploadService.UploadFiles(params...) } diff --git a/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/download.go b/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/download.go index 64bdc39e..a82375dd 100644 --- a/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/download.go +++ b/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/download.go @@ -577,7 +577,7 @@ func (ds *DownloadService) createFileHandlerFunc(downloadParams DownloadParams, } log.Info(fmt.Sprintf("%sDownloading %q to %q", logMsgPrefix, downloadData.Dependency.GetItemRelativePath(), localFullPath)) if err = ds.downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix, downloadData, downloadParams); err != nil { - log.Error(logMsgPrefix, "Received an error: "+err.Error()) + log.Error(logMsgPrefix + "Received an error: " + err.Error()) return err } successCounters[threadId]++ diff --git a/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/upload.go b/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/upload.go index 250b2267..75d9a0f2 100644 --- a/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/upload.go +++ b/vendor/github.com/jfrog/jfrog-client-go/artifactory/services/upload.go @@ -858,19 +858,22 @@ func (us *UploadService) CreateUploadAsZipFunc(uploadResult *utils.Result, targe } // Make sure all go routines in readFilesAsZip calls were done. var zipReadersWg sync.WaitGroup - checksumZipReader := us.readFilesAsZip(archiveDataReader, "Calculating size / checksums", + // We execute readFilesAsZip twice. The first execution is a dry-run. + // It calculates the size and checksums of the generated zip file, which is necessary before starting the upload task. + // The second execution is part of the zip upload process running in parallel to the archiving of them files to the zip. + checksumZipReader := us.readFilesAsZip(archiveDataReader, true, archiveData.uploadParams.Flat, archiveData.uploadParams.Symlink, saveFilesPathsFunc, errorsQueue, &zipReadersWg) details, err := fileutils.GetFileDetailsFromReader(checksumZipReader, archiveData.uploadParams.ChecksumsCalcEnabled) if err != nil { return } - log.Info(logMsgPrefix+"Uploading artifact:", targetPath) - getReaderFunc := func() (io.Reader, error) { archiveDataReader.Reset() - return us.readFilesAsZip(archiveDataReader, "Archiving", archiveData.uploadParams.Flat, + return us.readFilesAsZip(archiveDataReader, false, archiveData.uploadParams.Flat, archiveData.uploadParams.Symlink, nil, errorsQueue, &zipReadersWg), nil } + + log.Info(logMsgPrefix+"Uploading artifact:", targetPath) uploaded, err := us.uploadFileFromReader(getReaderFunc, targetUrlWithProps, archiveData.uploadParams, logMsgPrefix, details) if uploaded { @@ -886,12 +889,17 @@ func (us *UploadService) CreateUploadAsZipFunc(uploadResult *utils.Result, targe // Reads files and streams them as a ZIP to a Reader. // archiveDataReader is a ContentReader of UploadData items containing the details of the files to stream. +// zipDryRun - If true, this function is run as part of the dry-run process, to // saveFilesPathsFunc (optional) is a func that is called for each file that is written into the ZIP, and gets the file's local path as a parameter. -func (us *UploadService) readFilesAsZip(archiveDataReader *content.ContentReader, progressPrefix string, flat, symlink bool, +func (us *UploadService) readFilesAsZip(archiveDataReader *content.ContentReader, zipDryRun, flat, symlink bool, saveFilesPathsFunc func(sourcePath string) error, errorsQueue *clientutils.ErrorsQueue, zipReadersWg *sync.WaitGroup) io.Reader { pr, pw := io.Pipe() zipReadersWg.Add(1) + progressPrefix := "Archiving" + if zipDryRun { + progressPrefix = "Calculating size / checksums" + } go func() { defer zipReadersWg.Done() var e error @@ -912,6 +920,10 @@ func (us *UploadService) readFilesAsZip(archiveDataReader *content.ContentReader if e != nil { errorsQueue.AddError(e) } + if !zipDryRun && us.Progress != nil { + // Increment general progress by 1 for each file added to the zip if not in dry-run mode. + us.Progress.IncrementGeneralProgress() + } } if saveFilesPathsFunc != nil { e = saveFilesPathsFunc(uploadData.Artifact.LocalPath) @@ -979,7 +991,7 @@ func (us *UploadService) addFileToZip(artifact *clientutils.Artifact, progressPr err = errors.Join(err, errorutils.CheckError(file.Close())) } }() - // Show progress bar only for files larger than 250Kb to avoid polluting the terminal with endless progress bars. + // Show progress bar only for files larger than 500kb to avoid polluting the terminal with endless progress bars. if us.Progress != nil && info.Size() > minFileSizeForProgressInKb { progressReader := us.Progress.NewProgressReader(info.Size(), progressPrefix, localPath) reader = progressReader.ActionWithProgress(file) diff --git a/vendor/github.com/jfrog/jfrog-client-go/utils/utils.go b/vendor/github.com/jfrog/jfrog-client-go/utils/utils.go index fa4f188d..05ac1438 100644 --- a/vendor/github.com/jfrog/jfrog-client-go/utils/utils.go +++ b/vendor/github.com/jfrog/jfrog-client-go/utils/utils.go @@ -28,7 +28,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.46.2" + Version = "1.47.0" ) type MinVersionProduct string diff --git a/vendor/modules.txt b/vendor/modules.txt index 2d4d0b80..254ad2c0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -196,7 +196,7 @@ github.com/jedib0t/go-pretty/v6/text # github.com/jfrog/archiver/v3 v3.6.1 ## explicit; go 1.20 github.com/jfrog/archiver/v3 -# github.com/jfrog/build-info-go v1.9.36 +# github.com/jfrog/build-info-go v1.10.0 ## explicit; go 1.22 github.com/jfrog/build-info-go/build github.com/jfrog/build-info-go/build/utils @@ -220,8 +220,8 @@ github.com/jfrog/gofrog/safeconvert github.com/jfrog/gofrog/stringutils github.com/jfrog/gofrog/unarchive github.com/jfrog/gofrog/version -# github.com/jfrog/jfrog-cli-core/v2 v2.55.7 -## explicit; go 1.22.3 +# github.com/jfrog/jfrog-cli-core/v2 v2.56.0 +## explicit; go 1.22.7 github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils github.com/jfrog/jfrog-cli-core/v2/artifactory/utils @@ -237,8 +237,8 @@ github.com/jfrog/jfrog-cli-core/v2/utils/ioutils github.com/jfrog/jfrog-cli-core/v2/utils/lock github.com/jfrog/jfrog-cli-core/v2/utils/log github.com/jfrog/jfrog-cli-core/v2/utils/osutils -# github.com/jfrog/jfrog-client-go v1.46.2 -## explicit; go 1.22 +# github.com/jfrog/jfrog-client-go v1.47.0 +## explicit; go 1.22.7 github.com/jfrog/jfrog-client-go/access github.com/jfrog/jfrog-client-go/access/auth github.com/jfrog/jfrog-client-go/access/services @@ -441,15 +441,15 @@ golang.org/x/crypto/ssh golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts -# golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e -## explicit; go 1.20 +# golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 +## explicit; go 1.22.0 golang.org/x/exp/constraints golang.org/x/exp/maps golang.org/x/exp/slices golang.org/x/exp/slog golang.org/x/exp/slog/internal golang.org/x/exp/slog/internal/buffer -# golang.org/x/net v0.28.0 +# golang.org/x/net v0.29.0 ## explicit; go 1.18 golang.org/x/net/context golang.org/x/net/internal/socks