From b7891b0a3e7b017c520c1d4281651edfd0283edd Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:41:45 +0300 Subject: [PATCH 1/3] Audit Xray scan Sarif output (#996) --- xray/commands/scan/buildscan.go | 6 +- xray/commands/scan/scan.go | 2 +- xray/utils/resultwriter.go | 154 +++++++++++++++++++------------- xray/utils/resultwriter_test.go | 4 +- 4 files changed, 100 insertions(+), 66 deletions(-) diff --git a/xray/commands/scan/buildscan.go b/xray/commands/scan/buildscan.go index ec9f9a37e..d44a9dcb2 100644 --- a/xray/commands/scan/buildscan.go +++ b/xray/commands/scan/buildscan.go @@ -101,7 +101,7 @@ func (bsc *BuildScanCommand) Run() (err error) { Rescan: bsc.rescan, } - isFailBuildResponse, err := bsc.runBuildScanAndPrintResults(xrayManager, params) + isFailBuildResponse, err := bsc.runBuildScanAndPrintResults(xrayManager, xrayVersion, params) if err != nil { return err } @@ -112,7 +112,7 @@ func (bsc *BuildScanCommand) Run() (err error) { return } -func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayServicesManager, params services.XrayBuildParams) (isFailBuildResponse bool, err error) { +func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayServicesManager, xrayVersion string, params services.XrayBuildParams) (isFailBuildResponse bool, err error) { buildScanResults, noFailBuildPolicy, err := xrayManager.BuildScan(params, bsc.includeVulnerabilities) if err != nil { return false, err @@ -126,7 +126,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS XrayDataUrl: buildScanResults.MoreDetailsUrl, }} - extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse} + extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse, XrayVersion: xrayVersion} resultsPrinter := xrutils.NewResultsWriter(extendedScanResults). SetOutputFormat(bsc.outputFormat). diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index cd910ea0a..7a291a21c 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -241,7 +241,7 @@ func (scanCmd *ScanCommand) Run() (err error) { } scanErrors = appendErrorSlice(scanErrors, fileProducerErrors) scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors) - extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults} + extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults, XrayVersion: xrayVersion} if err = xrutils.NewResultsWriter(extendedScanResults). SetOutputFormat(scanCmd.outputFormat). diff --git a/xray/utils/resultwriter.go b/xray/utils/resultwriter.go index 553f37388..35a8fafd4 100644 --- a/xray/utils/resultwriter.go +++ b/xray/utils/resultwriter.go @@ -220,76 +220,89 @@ func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMult func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResults) error { for _, vulnerability := range xrayJson.Vulnerabilities { - if err := addXrayCveIssueToSarifRun( - vulnerability.Cves, - vulnerability.IssueId, - vulnerability.Severity, - vulnerability.Technology, - vulnerability.Components, - vulnerability.Applicable, - vulnerability.ImpactedDependencyName, - vulnerability.ImpactedDependencyVersion, - vulnerability.Summary, - vulnerability.FixedVersions, - run, - ); err != nil { + if err := addXrayCveIssueToSarifRun(vulnerability, run); err != nil { return err } } for _, violation := range xrayJson.SecurityViolations { - if err := addXrayCveIssueToSarifRun( - violation.Cves, - violation.IssueId, - violation.Severity, - violation.Technology, - violation.Components, - violation.Applicable, - violation.ImpactedDependencyName, - violation.ImpactedDependencyVersion, - violation.Summary, - violation.FixedVersions, - run, - ); err != nil { + if err := addXrayCveIssueToSarifRun(violation, run); err != nil { return err } } for _, license := range xrayJson.LicensesViolations { - msg := getVulnerabilityOrViolationSarifHeadline(license.LicenseKey, license.ImpactedDependencyName, license.ImpactedDependencyVersion) - if rule, isNewRule := addResultToSarifRun(license.LicenseKey, msg, license.Severity, nil, run); isNewRule { - rule.WithDescription("License watch violations") + if err := addXrayLicenseViolationToSarifRun(license, run); err != nil { + return err } } return nil } -func addXrayCveIssueToSarifRun(cves []formats.CveRow, issueId, severity string, tech coreutils.Technology, components []formats.ComponentRow, applicable, impactedDependencyName, impactedDependencyVersion, summary string, fixedVersions []string, run *sarif.Run) error { - maxCveScore, err := findMaxCVEScore(cves) +func addXrayCveIssueToSarifRun(issue formats.VulnerabilityOrViolationRow, run *sarif.Run) (err error) { + maxCveScore, err := findMaxCVEScore(issue.Cves) if err != nil { - return err + return } - cveId := GetIssueIdentifier(cves, issueId) - msg := getVulnerabilityOrViolationSarifHeadline(impactedDependencyName, impactedDependencyVersion, cveId) - location, err := getXrayIssueLocationIfValidExists(tech, run) + location, err := getXrayIssueLocationIfValidExists(issue.Technology, run) if err != nil { - return err + return } - if rule, isNewRule := addResultToSarifRun(cveId, msg, severity, location, run); isNewRule { - cveRuleProperties := sarif.NewPropertyBag() - if maxCveScore != MissingCveScore { - cveRuleProperties.Add("security-severity", maxCveScore) - } - rule.WithProperties(cveRuleProperties.Properties) - formattedDirectDependencies, err := getDirectDependenciesFormatted(components) - if err != nil { - return err + formattedDirectDependencies, err := getDirectDependenciesFormatted(issue.Components) + if err != nil { + return + } + cveId := GetIssueIdentifier(issue.Cves, issue.IssueId) + markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, issue.Applicable, issue.FixedVersions) + addXrayIssueToSarifRun( + cveId, + issue.ImpactedDependencyName, + issue.ImpactedDependencyVersion, + issue.Severity, + maxCveScore, + issue.Summary, + getXrayIssueSarifHeadline(issue.ImpactedDependencyName, issue.ImpactedDependencyVersion, cveId), + markdownDescription, + issue.Components, + location, + run, + ) + return +} + +func addXrayLicenseViolationToSarifRun(license formats.LicenseRow, run *sarif.Run) (err error) { + formattedDirectDependencies, err := getDirectDependenciesFormatted(license.Components) + if err != nil { + return + } + addXrayIssueToSarifRun( + license.LicenseKey, + license.ImpactedDependencyName, + license.ImpactedDependencyVersion, + license.Severity, + MissingCveScore, + getLicenseViolationSummary(license.ImpactedDependencyName, license.ImpactedDependencyVersion, license.LicenseKey), + getXrayLicenseSarifHeadline(license.ImpactedDependencyName, license.ImpactedDependencyVersion, license.LicenseKey), + getLicenseViolationMarkdown(license.ImpactedDependencyName, license.ImpactedDependencyVersion, license.LicenseKey, formattedDirectDependencies), + license.Components, + nil, + run, + ) + return +} + +func addXrayIssueToSarifRun(issueId, impactedDependencyName, impactedDependencyVersion, severity, severityScore, summary, title, markdownDescription string, components []formats.ComponentRow, location *sarif.Location, run *sarif.Run) { + // Add rule if not exists + ruleId := getXrayIssueSarifRuleId(impactedDependencyName, impactedDependencyVersion, issueId) + if rule, _ := run.GetRuleById(ruleId); rule == nil { + addXrayRule(ruleId, title, severityScore, summary, markdownDescription, run) + } + // Add result for each component + for _, directDependency := range components { + msg := getXrayIssueSarifHeadline(directDependency.Name, directDependency.Version, issueId) + if result := run.CreateResultForRule(ruleId).WithMessage(sarif.NewTextMessage(msg)).WithLevel(ConvertToSarifLevel(severity)); location != nil { + result.AddLocation(location) } - markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, applicable, fixedVersions) + "\n" - rule.WithHelp(&sarif.MultiformatMessageString{ - Text: &summary, - Markdown: &markdownDescription, - }) } - return nil + } func getDescriptorFullPath(tech coreutils.Technology, run *sarif.Run) (string, error) { @@ -322,15 +335,20 @@ func getXrayIssueLocationIfValidExists(tech coreutils.Technology, run *sarif.Run return sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + descriptorPath))), nil } -func addResultToSarifRun(issueId, msg, severity string, location *sarif.Location, run *sarif.Run) (rule *sarif.ReportingDescriptor, isNewRule bool) { - if rule, _ = run.GetRuleById(issueId); rule == nil { - isNewRule = true - rule = run.AddRule(issueId) - } - if result := run.CreateResultForRule(issueId).WithMessage(sarif.NewTextMessage(msg)).WithLevel(ConvertToSarifLevel(severity)); location != nil { - result.AddLocation(location) +func addXrayRule(ruleId, ruleDescription, maxCveScore, summary, markdownDescription string, run *sarif.Run) { + rule := run.AddRule(ruleId) + + if maxCveScore != MissingCveScore { + cveRuleProperties := sarif.NewPropertyBag() + cveRuleProperties.Add("security-severity", maxCveScore) + rule.WithProperties(cveRuleProperties.Properties) } - return + + rule.WithDescription(ruleDescription) + rule.WithHelp(&sarif.MultiformatMessageString{ + Text: &summary, + Markdown: &markdownDescription, + }) } func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { @@ -398,10 +416,26 @@ func GetIssueIdentifier(cvesRow []formats.CveRow, issueId string) string { return identifier } -func getVulnerabilityOrViolationSarifHeadline(depName, version, key string) string { +func getXrayIssueSarifRuleId(depName, version, key string) string { + return fmt.Sprintf("%s_%s_%s", key, depName, version) +} + +func getXrayIssueSarifHeadline(depName, version, key string) string { return fmt.Sprintf("[%s] %s %s", key, depName, version) } +func getXrayLicenseSarifHeadline(depName, version, key string) string { + return fmt.Sprintf("License violation [%s] %s %s", key, depName, version) +} + +func getLicenseViolationSummary(depName, version, key string) string { + return fmt.Sprintf("Dependency %s version %s is using a license (%s) that is not allowed.", depName, version, key) +} + +func getLicenseViolationMarkdown(depName, version, key, formattedDirectDependencies string) string { + return fmt.Sprintf("**The following direct dependencies are utilizing the `%s %s` dependency with `%s` license violation:**\n%s", depName, version, key, formattedDirectDependencies) +} + func getDirectDependenciesFormatted(directDependencies []formats.ComponentRow) (string, error) { var formattedDirectDependencies strings.Builder for _, dependency := range directDependencies { diff --git a/xray/utils/resultwriter_test.go b/xray/utils/resultwriter_test.go index a67dfe86c..b6fa06d11 100644 --- a/xray/utils/resultwriter_test.go +++ b/xray/utils/resultwriter_test.go @@ -13,8 +13,8 @@ import ( ) func TestGetVulnerabilityOrViolationSarifHeadline(t *testing.T) { - assert.Equal(t, "[CVE-2022-1234] loadsh 1.4.1", getVulnerabilityOrViolationSarifHeadline("loadsh", "1.4.1", "CVE-2022-1234")) - assert.NotEqual(t, "[CVE-2022-1234] loadsh 1.4.1", getVulnerabilityOrViolationSarifHeadline("loadsh", "1.2.1", "CVE-2022-1234")) + assert.Equal(t, "[CVE-2022-1234] loadsh 1.4.1", getXrayIssueSarifHeadline("loadsh", "1.4.1", "CVE-2022-1234")) + assert.NotEqual(t, "[CVE-2022-1234] loadsh 1.4.1", getXrayIssueSarifHeadline("loadsh", "1.2.1", "CVE-2022-1234")) } func TestGetIssueIdentifier(t *testing.T) { From d60d14980955de0f191051b1fe50b2ac4dc560e1 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Wed, 18 Oct 2023 10:13:41 +0300 Subject: [PATCH 2/3] Transfer - Fix phase 3 progressbar (#999) --- .../transferfiles/delayedartifactshandler.go | 12 ++++++----- .../delayedartifactshandler_test.go | 16 ++++++++------- .../commands/transferfiles/fileserror.go | 20 ++++++++++++++++--- artifactory/commands/transferfiles/status.go | 2 +- .../commands/transferfiles/status_test.go | 2 +- .../progressbar/transferprogressbarmanager.go | 2 +- 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index 693babc20..277cbe503 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -120,7 +120,7 @@ func consumeDelayFilesIfNoErrors(phase phaseBase, addedDelayFiles []string) erro if len(addedDelayFiles) > 0 && phase.progressBar != nil { phaseTaskProgressBar := phase.progressBar.phases[phase.phaseId].GetTasksProgressBar() oldTotal := phaseTaskProgressBar.GetTotal() - delayCount, err := countDelayFilesContent(addedDelayFiles) + delayCount, _, err := countDelayFilesContent(addedDelayFiles) if err != nil { return err } @@ -129,16 +129,18 @@ func consumeDelayFilesIfNoErrors(phase phaseBase, addedDelayFiles []string) erro return nil } -func countDelayFilesContent(filePaths []string) (int, error) { - count := 0 +func countDelayFilesContent(filePaths []string) (count int, storage int64, err error) { for _, file := range filePaths { delayFile, err := readDelayFile(file) if err != nil { - return 0, err + return 0, storage, err } count += len(delayFile.DelayedArtifacts) + for _, delay := range delayFile.DelayedArtifacts { + storage += delay.Size + } } - return count, nil + return } func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayUploadComparisonFunctions []shouldDelayUpload) error { diff --git a/artifactory/commands/transferfiles/delayedartifactshandler_test.go b/artifactory/commands/transferfiles/delayedartifactshandler_test.go index cd576f9e3..ebff3bfab 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler_test.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler_test.go @@ -2,17 +2,18 @@ package transferfiles import ( "fmt" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" - "github.com/jfrog/jfrog-cli-core/v2/utils/tests" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - "github.com/stretchr/testify/assert" "math" "os" "path/filepath" "sync" "testing" "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" + "github.com/jfrog/jfrog-cli-core/v2/utils/tests" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/stretchr/testify/assert" ) var delayTestRepoKey = "delay-local-repo" @@ -52,7 +53,7 @@ func TestDelayedArtifactsMng(t *testing.T) { go func() { defer writeWaitGroup.Done() for i := 0; i < artifactsNumber; i++ { - artifactsChannelMng.channel <- api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("name%d", i)} + artifactsChannelMng.channel <- api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("name%d", i), Size: int64(i)} } }() @@ -71,8 +72,9 @@ func TestDelayedArtifactsMng(t *testing.T) { expectedNumberOfFiles := int(math.Ceil(float64(artifactsNumber) / float64(maxDelayedArtifactsInFile))) validateDelayedArtifactsFiles(t, delayFiles, expectedNumberOfFiles, artifactsNumber) - delayCount, err := countDelayFilesContent(delayFiles) + delayCount, storage, err := countDelayFilesContent(delayFiles) assert.NoError(t, err) + assert.Equal(t, int64(1225), storage) assert.Equal(t, delayCount, artifactsNumber) } diff --git a/artifactory/commands/transferfiles/fileserror.go b/artifactory/commands/transferfiles/fileserror.go index 7cf746c04..0a74b5e75 100644 --- a/artifactory/commands/transferfiles/fileserror.go +++ b/artifactory/commands/transferfiles/fileserror.go @@ -3,13 +3,14 @@ package transferfiles import ( "errors" "fmt" + "os" + "strings" + "time" + "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" - "os" - "strings" - "time" ) type errorFileHandlerFunc func() parallel.TaskFunc @@ -150,6 +151,19 @@ func (e *errorsRetryPhase) initProgressBar() error { filesCount += len(failedFiles.Errors) } // The progress bar will also be responsible to display the number of delayed items for this repository. + // Those delayed artifacts will be handled at the end of this phase in case they exist. + delayFiles, err := getDelayFiles([]string{e.repoKey}) + if err != nil { + return err + } + delayCount, delayStorage, err := countDelayFilesContent(delayFiles) + if err != nil { + return err + } + err = e.stateManager.SetTotalSizeAndFilesPhase3(int64(filesCount)+int64(delayCount), storage+delayStorage) + if err != nil { + return err + } e.progressBar.AddPhase3() return nil } diff --git a/artifactory/commands/transferfiles/status.go b/artifactory/commands/transferfiles/status.go index 0ffd1d6be..4250542b4 100644 --- a/artifactory/commands/transferfiles/status.go +++ b/artifactory/commands/transferfiles/status.go @@ -102,7 +102,7 @@ func setRepositoryStatus(stateManager *state.TransferStateManager, output *strin if stateManager.CurrentRepoPhase == api.Phase1 { addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 3) } else { - addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 3) + addString(output, "🔢", "Phase", "Retrying transfer failures and transfer delayed files (3/3)", 3) } addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 3) addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 3) diff --git a/artifactory/commands/transferfiles/status_test.go b/artifactory/commands/transferfiles/status_test.go index 37be3b113..563d7490f 100644 --- a/artifactory/commands/transferfiles/status_test.go +++ b/artifactory/commands/transferfiles/status_test.go @@ -138,7 +138,7 @@ func TestShowBuildInfoRepo(t *testing.T) { // Check repository status assert.Contains(t, results, "Current Repository Status") assert.Contains(t, results, "Name: repo1") - assert.Contains(t, results, "Phase: Retrying transfer failures (3/3)") + assert.Contains(t, results, "Phase: Retrying transfer failures and transfer delayed files (3/3)") assert.Contains(t, results, "Delayed files: 20 (Files to be transferred last, after all other files)") assert.Contains(t, results, "Storage: 4.9 KiB / 9.8 KiB (50.0%)") assert.Contains(t, results, "Files: 500 / 10000 (5.0%)") diff --git a/utils/progressbar/transferprogressbarmanager.go b/utils/progressbar/transferprogressbarmanager.go index 840ccd02d..d479998b4 100644 --- a/utils/progressbar/transferprogressbarmanager.go +++ b/utils/progressbar/transferprogressbarmanager.go @@ -15,7 +15,7 @@ import ( const ( phase1HeadLine = "Phase 1: Transferring all files in the repository" phase2HeadLine = "Phase 2: Transferring newly created and modified files" - phase3HeadLine = "Phase 3: Retrying transfer failures" + phase3HeadLine = "Phase 3: Retrying transfer failures and transfer delayed files" DelayedFilesContentNote = "Files to be transferred last, after all other files" RetryFailureContentNote = "In Phase 3 and in subsequent executions, we'll retry transferring the failed files" ) From b1e98b93cfd76592a8c1608defbb4bfb39788024 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Wed, 18 Oct 2023 10:25:06 +0300 Subject: [PATCH 3/3] Transfer - Progress should only represent successful storage transfer (#1000) --- utils/progressbar/progressbarmng.go | 39 ++++++++----------- .../progressbar/transferprogressbarmanager.go | 9 ++--- 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/utils/progressbar/progressbarmng.go b/utils/progressbar/progressbarmng.go index d9e7ff010..2c9dce525 100644 --- a/utils/progressbar/progressbarmng.go +++ b/utils/progressbar/progressbarmng.go @@ -95,18 +95,17 @@ func (bm *ProgressBarMng) newDoubleValueProgressBar(getVal func() (firstNumerato if err != nil { log.Error(err) } - numeratorString := artifactoryutils.ConvertIntToStorageSizeString(getProgressNumerator(statistics, *firstNumerator, *firstDenominator)) - denominatorString := artifactoryutils.ConvertIntToStorageSizeString(*firstDenominator) - return color.Green.Render(numeratorString + "/" + denominatorString) - }), - decor.Name(" "+secondValueLine+": "), decor.Any(func(statistics decor.Statistics) string { + s1 := artifactoryutils.ConvertIntToStorageSizeString(*firstNumerator) + s2 := artifactoryutils.ConvertIntToStorageSizeString(*firstDenominator) + return color.Green.Render(s1 + "/" + s2) + }), decor.Name(" "+secondValueLine+": "), decor.Any(func(statistics decor.Statistics) string { _, _, secondNumerator, secondDenominator, err := getVal() if err != nil { log.Error(err) } - numeratorString := strconv.FormatInt(getProgressNumerator(statistics, *secondNumerator, *secondDenominator), 10) - denominatorString := strconv.Itoa(int(*secondDenominator)) - return color.Green.Render(numeratorString + "/" + denominatorString) + s1 := strconv.Itoa(int(*secondNumerator)) + s2 := strconv.Itoa(int(*secondDenominator)) + return color.Green.Render(s1 + "/" + s2) }), ), ) @@ -189,9 +188,7 @@ func (bm *ProgressBarMng) NewHeadlineBar(msg string) *mpb.Bar { func (bm *ProgressBarMng) Increment(prog *TasksWithHeadlineProg) { bm.barsRWMutex.RLock() defer bm.barsRWMutex.RUnlock() - if prog.tasksProgressBar.bar.Current() < math.MaxInt64 { - prog.tasksProgressBar.bar.Increment() - } + prog.tasksProgressBar.bar.Increment() prog.tasksProgressBar.tasksCount++ } @@ -199,9 +196,7 @@ func (bm *ProgressBarMng) Increment(prog *TasksWithHeadlineProg) { func (bm *ProgressBarMng) IncBy(n int, prog *TasksWithHeadlineProg) { bm.barsRWMutex.RLock() defer bm.barsRWMutex.RUnlock() - if prog.tasksProgressBar.bar.Current() < math.MaxInt64 { - prog.tasksProgressBar.bar.IncrBy(n) - } + prog.tasksProgressBar.bar.IncrBy(n) prog.tasksProgressBar.tasksCount += int64(n) } @@ -209,7 +204,12 @@ func (bm *ProgressBarMng) IncBy(n int, prog *TasksWithHeadlineProg) { func (bm *ProgressBarMng) DoneTask(prog *TasksWithHeadlineProg) { bm.barsRWMutex.RLock() defer bm.barsRWMutex.RUnlock() - prog.tasksProgressBar.bar.SetCurrent(math.MaxInt64) + diff := prog.tasksProgressBar.total - prog.tasksProgressBar.tasksCount + // diff is int64, but we can increase the progress up to math.MaxInt in a time + for ; diff > math.MaxInt; diff -= math.MaxInt { + prog.tasksProgressBar.bar.IncrBy(math.MaxInt) + } + prog.tasksProgressBar.bar.IncrBy(int(diff)) } func (bm *ProgressBarMng) NewTasksProgressBar(totalTasks int64, windows bool, taskType string) *TasksProgressBar { @@ -240,7 +240,7 @@ func (bm *ProgressBarMng) newTasksProgressBar(getVal func() (numerator, denomina mpb.AppendDecorators( decor.Name(" "+headLine+": "), decor.Any(func(statistics decor.Statistics) string { - numeratorString := strconv.FormatInt(getProgressNumerator(statistics, *numerator, *denominator), 10) + numeratorString := strconv.Itoa(int(*numerator)) denominatorString := strconv.Itoa(int(*denominator)) return color.Green.Render(numeratorString + "/" + denominatorString) }), @@ -353,10 +353,3 @@ func setTerminalWidthVar() error { } return err } - -func getProgressNumerator(statistics decor.Statistics, numerator, denominator int64) int64 { - if statistics.Current == math.MaxInt64 { - return denominator - } - return numerator -} diff --git a/utils/progressbar/transferprogressbarmanager.go b/utils/progressbar/transferprogressbarmanager.go index d479998b4..340166dc2 100644 --- a/utils/progressbar/transferprogressbarmanager.go +++ b/utils/progressbar/transferprogressbarmanager.go @@ -1,7 +1,6 @@ package progressbar import ( - "math" "sync" "time" @@ -118,7 +117,7 @@ func (tpm *TransferProgressMng) NewPhase1ProgressBar() *TasksWithHeadlineProg { if tpm.currentRepoShouldStop { return } - transferredStorage, totalStorage, _, _, err := getVals() + ptr1, ptr2, _, _, err := getVals() if err != nil { log.Error("Error: Couldn't get needed information about transfer status from state") } @@ -127,10 +126,8 @@ func (tpm *TransferProgressMng) NewPhase1ProgressBar() *TasksWithHeadlineProg { return } if pb.GetTasksProgressBar() != nil { - pb.GetTasksProgressBar().SetGeneralProgressTotal(*totalStorage) - if pb.GetTasksProgressBar().GetBar().Current() < math.MaxInt64 { - pb.GetTasksProgressBar().GetBar().SetCurrent(*transferredStorage) - } + pb.GetTasksProgressBar().SetGeneralProgressTotal(*ptr2) + pb.GetTasksProgressBar().GetBar().SetCurrent(*ptr1) } time.Sleep(1 * time.Second) }