Merge remote-tracking branch 'origin/dev'

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -120,7 +120,7 @@ func consumeDelayFilesIfNoErrors(phase phaseBase, addedDelayFiles []string) erro
  if len(addedDelayFiles) > 0 && phase.progressBar != nil {
  phaseTaskProgressBar := phase.progressBar.phases[phase.phaseId].GetTasksProgressBar()
  oldTotal := phaseTaskProgressBar.GetTotal()
- delayCount, err := countDelayFilesContent(addedDelayFiles)
+ delayCount, _, err := countDelayFilesContent(addedDelayFiles)
  if err != nil {
  return err
  }
@@ -129,16 +129,18 @@ func consumeDelayFilesIfNoErrors(phase phaseBase, addedDelayFiles []string) erro
  return nil
 }
 
-func countDelayFilesContent(filePaths []string) (int, error) {
- count := 0
+func countDelayFilesContent(filePaths []string) (count int, storage int64, err error) {
  for _, file := range filePaths {
  delayFile, err := readDelayFile(file)
  if err != nil {
- return 0, err
+ return 0, storage, err
  }
  count += len(delayFile.DelayedArtifacts)
+ for _, delay := range delayFile.DelayedArtifacts {
+ storage += delay.Size
+ }
  }
- return count, nil
+ return
 }
 
 func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayUploadComparisonFunctions []shouldDelayUpload) error {

artifactory/commands/transferfiles/delayedartifactshandler_test.go

@@ -2,17 +2,18 @@ package transferfiles
 
 import (
  "fmt"
- "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
- "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
- "github.com/jfrog/jfrog-cli-core/v2/utils/tests"
- "github.com/jfrog/jfrog-client-go/utils/io/fileutils"
- "github.com/stretchr/testify/assert"
  "math"
  "os"
  "path/filepath"
  "sync"
  "testing"
  "time"
+
+ "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
+ "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
+ "github.com/jfrog/jfrog-cli-core/v2/utils/tests"
+ "github.com/jfrog/jfrog-client-go/utils/io/fileutils"
+ "github.com/stretchr/testify/assert"
 )
 
 var delayTestRepoKey = "delay-local-repo"
@@ -52,7 +53,7 @@ func TestDelayedArtifactsMng(t *testing.T) {
  go func() {
  defer writeWaitGroup.Done()
  for i := 0; i < artifactsNumber; i++ {
- artifactsChannelMng.channel <- api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("name%d", i)}
+ artifactsChannelMng.channel <- api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("name%d", i), Size: int64(i)}
  }
  }()
 
@@ -71,8 +72,9 @@ func TestDelayedArtifactsMng(t *testing.T) {
  expectedNumberOfFiles := int(math.Ceil(float64(artifactsNumber) / float64(maxDelayedArtifactsInFile)))
  validateDelayedArtifactsFiles(t, delayFiles, expectedNumberOfFiles, artifactsNumber)
 
- delayCount, err := countDelayFilesContent(delayFiles)
+ delayCount, storage, err := countDelayFilesContent(delayFiles)
  assert.NoError(t, err)
+ assert.Equal(t, int64(1225), storage)
  assert.Equal(t, delayCount, artifactsNumber)
 }
 

artifactory/commands/transferfiles/fileserror.go

@@ -3,13 +3,14 @@ package transferfiles
 import (
  "errors"
  "fmt"
+ "os"
+ "strings"
+ "time"
+
  "github.com/jfrog/gofrog/parallel"
  "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
  "github.com/jfrog/jfrog-client-go/utils/errorutils"
  "github.com/jfrog/jfrog-client-go/utils/log"
- "os"
- "strings"
- "time"
 )
 
 type errorFileHandlerFunc func() parallel.TaskFunc
@@ -150,6 +151,19 @@ func (e *errorsRetryPhase) initProgressBar() error {
  filesCount += len(failedFiles.Errors)
  }
  // The progress bar will also be responsible to display the number of delayed items for this repository.
+ // Those delayed artifacts will be handled at the end of this phase in case they exist.
+ delayFiles, err := getDelayFiles([]string{e.repoKey})
+ if err != nil {
+ return err
+ }
+ delayCount, delayStorage, err := countDelayFilesContent(delayFiles)
+ if err != nil {
+ return err
+ }
+ err = e.stateManager.SetTotalSizeAndFilesPhase3(int64(filesCount)+int64(delayCount), storage+delayStorage)
+ if err != nil {
+ return err
+ }
  e.progressBar.AddPhase3()
  return nil
 }

artifactory/commands/transferfiles/status.go

@@ -102,7 +102,7 @@ func setRepositoryStatus(stateManager *state.TransferStateManager, output *strin
  if stateManager.CurrentRepoPhase == api.Phase1 {
  addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 3)
  } else {
- addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 3)
+ addString(output, "🔢", "Phase", "Retrying transfer failures and transfer delayed files (3/3)", 3)
  }
  addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 3)
  addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 3)

artifactory/commands/transferfiles/status_test.go

@@ -138,7 +138,7 @@ func TestShowBuildInfoRepo(t *testing.T) {
  // Check repository status
  assert.Contains(t, results, "Current Repository Status")
  assert.Contains(t, results, "Name: repo1")
- assert.Contains(t, results, "Phase: Retrying transfer failures (3/3)")
+ assert.Contains(t, results, "Phase: Retrying transfer failures and transfer delayed files (3/3)")
  assert.Contains(t, results, "Delayed files: 20 (Files to be transferred last, after all other files)")
  assert.Contains(t, results, "Storage: 4.9 KiB / 9.8 KiB (50.0%)")
  assert.Contains(t, results, "Files: 500 / 10000 (5.0%)")

utils/progressbar/progressbarmng.go

@@ -95,18 +95,17 @@ func (bm *ProgressBarMng) newDoubleValueProgressBar(getVal func() (firstNumerato
  if err != nil {
  log.Error(err)
  }
- numeratorString := artifactoryutils.ConvertIntToStorageSizeString(getProgressNumerator(statistics, *firstNumerator, *firstDenominator))
- denominatorString := artifactoryutils.ConvertIntToStorageSizeString(*firstDenominator)
- return color.Green.Render(numeratorString + "/" + denominatorString)
- }),
- decor.Name(" "+secondValueLine+": "), decor.Any(func(statistics decor.Statistics) string {
+ s1 := artifactoryutils.ConvertIntToStorageSizeString(*firstNumerator)
+ s2 := artifactoryutils.ConvertIntToStorageSizeString(*firstDenominator)
+ return color.Green.Render(s1 + "/" + s2)
+ }), decor.Name(" "+secondValueLine+": "), decor.Any(func(statistics decor.Statistics) string {
  _, _, secondNumerator, secondDenominator, err := getVal()
  if err != nil {
  log.Error(err)
  }
- numeratorString := strconv.FormatInt(getProgressNumerator(statistics, *secondNumerator, *secondDenominator), 10)
- denominatorString := strconv.Itoa(int(*secondDenominator))
- return color.Green.Render(numeratorString + "/" + denominatorString)
+ s1 := strconv.Itoa(int(*secondNumerator))
+ s2 := strconv.Itoa(int(*secondDenominator))
+ return color.Green.Render(s1 + "/" + s2)
  }),
  ),
  )
@@ -189,27 +188,28 @@ func (bm *ProgressBarMng) NewHeadlineBar(msg string) *mpb.Bar {
 func (bm *ProgressBarMng) Increment(prog *TasksWithHeadlineProg) {
  bm.barsRWMutex.RLock()
  defer bm.barsRWMutex.RUnlock()
- if prog.tasksProgressBar.bar.Current() < math.MaxInt64 {
- prog.tasksProgressBar.bar.Increment()
- }
+ prog.tasksProgressBar.bar.Increment()
  prog.tasksProgressBar.tasksCount++
 }
 
 // Increment increments completed tasks count by n.
 func (bm *ProgressBarMng) IncBy(n int, prog *TasksWithHeadlineProg) {
  bm.barsRWMutex.RLock()
  defer bm.barsRWMutex.RUnlock()
- if prog.tasksProgressBar.bar.Current() < math.MaxInt64 {
- prog.tasksProgressBar.bar.IncrBy(n)
- }
+ prog.tasksProgressBar.bar.IncrBy(n)
  prog.tasksProgressBar.tasksCount += int64(n)
 }
 
 // DoneTask increase tasks counter to the number of totalTasks.
 func (bm *ProgressBarMng) DoneTask(prog *TasksWithHeadlineProg) {
  bm.barsRWMutex.RLock()
  defer bm.barsRWMutex.RUnlock()
- prog.tasksProgressBar.bar.SetCurrent(math.MaxInt64)
+ diff := prog.tasksProgressBar.total - prog.tasksProgressBar.tasksCount
+ // diff is int64, but we can increase the progress up to math.MaxInt in a time
+ for ; diff > math.MaxInt; diff -= math.MaxInt {
+ prog.tasksProgressBar.bar.IncrBy(math.MaxInt)
+ }
+ prog.tasksProgressBar.bar.IncrBy(int(diff))
 }
 
 func (bm *ProgressBarMng) NewTasksProgressBar(totalTasks int64, windows bool, taskType string) *TasksProgressBar {
@@ -240,7 +240,7 @@ func (bm *ProgressBarMng) newTasksProgressBar(getVal func() (numerator, denomina
  mpb.AppendDecorators(
  decor.Name(" "+headLine+": "),
  decor.Any(func(statistics decor.Statistics) string {
- numeratorString := strconv.FormatInt(getProgressNumerator(statistics, *numerator, *denominator), 10)
+ numeratorString := strconv.Itoa(int(*numerator))
  denominatorString := strconv.Itoa(int(*denominator))
  return color.Green.Render(numeratorString + "/" + denominatorString)
  }),
@@ -353,10 +353,3 @@ func setTerminalWidthVar() error {
  }
  return err
 }
-
-func getProgressNumerator(statistics decor.Statistics, numerator, denominator int64) int64 {
- if statistics.Current == math.MaxInt64 {
- return denominator
- }
- return numerator
-}

utils/progressbar/transferprogressbarmanager.go

@@ -1,7 +1,6 @@
 package progressbar
 
 import (
- "math"
  "sync"
  "time"
 
@@ -15,7 +14,7 @@ import (
 const (
  phase1HeadLine = "Phase 1: Transferring all files in the repository"
  phase2HeadLine = "Phase 2: Transferring newly created and modified files"
- phase3HeadLine = "Phase 3: Retrying transfer failures"
+ phase3HeadLine = "Phase 3: Retrying transfer failures and transfer delayed files"
  DelayedFilesContentNote = "Files to be transferred last, after all other files"
  RetryFailureContentNote = "In Phase 3 and in subsequent executions, we'll retry transferring the failed files"
 )
@@ -118,7 +117,7 @@ func (tpm *TransferProgressMng) NewPhase1ProgressBar() *TasksWithHeadlineProg {
  if tpm.currentRepoShouldStop {
  return
  }
- transferredStorage, totalStorage, _, _, err := getVals()
+ ptr1, ptr2, _, _, err := getVals()
  if err != nil {
  log.Error("Error: Couldn't get needed information about transfer status from state")
  }
@@ -127,10 +126,8 @@ func (tpm *TransferProgressMng) NewPhase1ProgressBar() *TasksWithHeadlineProg {
  return
  }
  if pb.GetTasksProgressBar() != nil {
- pb.GetTasksProgressBar().SetGeneralProgressTotal(*totalStorage)
- if pb.GetTasksProgressBar().GetBar().Current() < math.MaxInt64 {
- pb.GetTasksProgressBar().GetBar().SetCurrent(*transferredStorage)
- }
+ pb.GetTasksProgressBar().SetGeneralProgressTotal(*ptr2)
+ pb.GetTasksProgressBar().GetBar().SetCurrent(*ptr1)
  }
  time.Sleep(1 * time.Second)
  }

xray/commands/scan/buildscan.go

@@ -101,7 +101,7 @@ func (bsc *BuildScanCommand) Run() (err error) {
  Rescan: bsc.rescan,
  }
 
- isFailBuildResponse, err := bsc.runBuildScanAndPrintResults(xrayManager, params)
+ isFailBuildResponse, err := bsc.runBuildScanAndPrintResults(xrayManager, xrayVersion, params)
  if err != nil {
  return err
  }
@@ -112,7 +112,7 @@ func (bsc *BuildScanCommand) Run() (err error) {
  return
 }
 
-func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayServicesManager, params services.XrayBuildParams) (isFailBuildResponse bool, err error) {
+func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayServicesManager, xrayVersion string, params services.XrayBuildParams) (isFailBuildResponse bool, err error) {
  buildScanResults, noFailBuildPolicy, err := xrayManager.BuildScan(params, bsc.includeVulnerabilities)
  if err != nil {
  return false, err
@@ -126,7 +126,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS
  XrayDataUrl: buildScanResults.MoreDetailsUrl,
  }}
 
- extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse}
+ extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse, XrayVersion: xrayVersion}
 
  resultsPrinter := xrutils.NewResultsWriter(extendedScanResults).
  SetOutputFormat(bsc.outputFormat).

xray/commands/scan/scan.go

@@ -241,7 +241,7 @@ func (scanCmd *ScanCommand) Run() (err error) {
  }
  scanErrors = appendErrorSlice(scanErrors, fileProducerErrors)
  scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors)
- extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults}
+ extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults, XrayVersion: xrayVersion}
 
  if err = xrutils.NewResultsWriter(extendedScanResults).
  SetOutputFormat(scanCmd.outputFormat).