Merge remote-tracking branch 'origin/dev'

go.mod

@@ -24,7 +24,6 @@ require (
  github.com/stretchr/testify v1.8.4
  github.com/urfave/cli v1.22.14
  github.com/vbauerster/mpb/v7 v7.5.3
- github.com/wagoodman/dive v0.11.0
  golang.org/x/exp v0.0.0-20230905200255-921286631fa9
  golang.org/x/mod v0.12.0
  golang.org/x/sync v0.3.0
@@ -43,28 +42,16 @@ require (
  github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
  github.com/acomagu/bufpipe v1.0.4 // indirect
  github.com/andybalholm/brotli v1.0.1 // indirect
- github.com/awesome-gocui/gocui v1.1.0 // indirect
- github.com/cespare/xxhash v1.1.0 // indirect
  github.com/cloudflare/circl v1.3.3 // indirect
  github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
  github.com/cyphar/filepath-securejoin v0.2.4 // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/docker/cli v24.0.5+incompatible // indirect
- github.com/docker/distribution v2.8.2+incompatible // indirect
- github.com/docker/docker v24.0.2+incompatible // indirect
- github.com/docker/go-connections v0.4.0 // indirect
- github.com/docker/go-units v0.4.0 // indirect
  github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
- github.com/dustin/go-humanize v1.0.0 // indirect
  github.com/emirpasic/gods v1.18.1 // indirect
- github.com/fatih/color v1.13.0 // indirect
  github.com/fsnotify/fsnotify v1.6.0 // indirect
- github.com/gdamore/encoding v1.0.0 // indirect
- github.com/gdamore/tcell/v2 v2.4.0 // indirect
  github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
  github.com/go-git/go-billy/v5 v5.5.0 // indirect
  github.com/go-git/go-git/v5 v5.9.0 // indirect
- github.com/gogo/protobuf v1.3.2 // indirect
  github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
  github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
  github.com/golang/snappy v0.0.2 // indirect
@@ -74,8 +61,6 @@ require (
  github.com/klauspost/compress v1.11.4 // indirect
  github.com/klauspost/cpuid/v2 v2.2.3 // indirect
  github.com/klauspost/pgzip v1.2.5 // indirect
- github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b // indirect
- github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
  github.com/mattn/go-colorable v0.1.12 // indirect
  github.com/mattn/go-isatty v0.0.14 // indirect
  github.com/mattn/go-runewidth v0.0.13 // indirect
@@ -84,19 +69,14 @@ require (
  github.com/minio/sha256-simd v1.0.1 // indirect
  github.com/mitchellh/mapstructure v1.5.0 // indirect
  github.com/nwaples/rardecode v1.1.0 // indirect
- github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
- github.com/opencontainers/image-spec v1.0.2 // indirect
  github.com/pelletier/go-toml/v2 v2.0.8 // indirect
- github.com/phayes/permbits v0.0.0-20190612203442-39d7c581d2ee // indirect
  github.com/pierrec/lz4/v4 v4.1.2 // indirect
  github.com/pjbgf/sha1cd v0.3.0 // indirect
- github.com/pkg/errors v0.9.1 // indirect
  github.com/pkg/term v1.1.0 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
  github.com/rivo/uniseg v0.4.3 // indirect
  github.com/russross/blackfriday/v2 v2.1.0 // indirect
  github.com/sergi/go-diff v1.1.0 // indirect
- github.com/sirupsen/logrus v1.9.0 // indirect
  github.com/skeema/knownhosts v1.2.0 // indirect
  github.com/spf13/afero v1.9.5 // indirect
  github.com/spf13/cast v1.5.1 // indirect

xray/commands/scan/buildscan.go

@@ -4,6 +4,7 @@ import (
  "errors"
  rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
  "github.com/jfrog/jfrog-cli-core/v2/utils/config"
+ "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
  xrutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
  clientutils "github.com/jfrog/jfrog-client-go/utils"
  "github.com/jfrog/jfrog-client-go/utils/log"
@@ -71,7 +72,7 @@ func (bsc *BuildScanCommand) SetRescan(rescan bool) *BuildScanCommand {
 
 // Scan published builds with Xray
 func (bsc *BuildScanCommand) Run() (err error) {
- xrayManager, xrayVersion, err := xrutils.CreateXrayServiceManagerAndGetVersion(bsc.serverDetails)
+ xrayManager, xrayVersion, err := utils.CreateXrayServiceManagerAndGetVersion(bsc.serverDetails)
  if err != nil {
  return err
  }

xray/commands/scan/dockerscan.go

@@ -9,7 +9,6 @@ import (
  "github.com/jfrog/jfrog-client-go/utils/errorutils"
  "github.com/jfrog/jfrog-client-go/utils/io/fileutils"
  "github.com/jfrog/jfrog-client-go/utils/log"
- "github.com/wagoodman/dive/dive"
  "os"
  "os/exec"
  "path/filepath"
@@ -19,17 +18,12 @@ import (
 const (
  indexerEnvPrefix = "JFROG_INDEXER_"
  DockerScanMinXrayVersion = "3.40.0"
- layerDigestPrefix = "sha256:"
- // Suffix added while analyzing docker layers, remove it for better readability.
- buildKitSuffix = " # buildkit"
 )
 
 type DockerScanCommand struct {
  ScanCommand
  imageTag string
  targetRepoPath string
- // Maps layer hash to dockerfile command
- dockerfileCommandsMapping map[string]string
 }
 
 func NewDockerScanCommand() *DockerScanCommand {
@@ -46,7 +40,6 @@ func (dsc *DockerScanCommand) SetTargetRepoPath(repoPath string) *DockerScanComm
  return dsc
 }
 
-// DockerScan scan will save a docker image as .tar file and will prefore binary scan on it.
 func (dsc *DockerScanCommand) Run() (err error) {
  // Validate Xray minimum version
  _, xrayVersion, err := xrayutils.CreateXrayServiceManagerAndGetVersion(dsc.ScanCommand.serverDetails)
@@ -75,26 +68,21 @@ func (dsc *DockerScanCommand) Run() (err error) {
  }
  log.Info("Creating image archive...")
  imageTarPath := filepath.Join(tempDirPath, "image.tar")
-
  dockerSaveCmd := exec.Command("docker", "save", dsc.imageTag, "-o", imageTarPath)
  var stderr bytes.Buffer
  dockerSaveCmd.Stderr = &stderr
- if err = dockerSaveCmd.Run(); err != nil {
+ err = dockerSaveCmd.Run()
+ if err != nil {
  return fmt.Errorf("failed running command: '%s' with error: %s - %s", strings.Join(dockerSaveCmd.Args, " "), err.Error(), stderr.String())
  }
 
- // Map layers sha256 checksum names to dockerfile line commands
- if err = dsc.mapDockerLayerToCommand(); err != nil {
- return
- }
-
  // Perform scan on image.tar
  dsc.SetSpec(spec.NewBuilder().
  Pattern(imageTarPath).
  Target(dsc.targetRepoPath).
  BuildSpec()).SetThreads(1)
-
- if err = dsc.setCredentialEnvsForIndexerApp(); err != nil {
+ err = dsc.setCredentialEnvsForIndexerApp()
+ if err != nil {
  return errorutils.CheckError(err)
  }
  defer func() {
@@ -103,56 +91,7 @@ func (dsc *DockerScanCommand) Run() (err error) {
  err = errorutils.CheckError(e)
  }
  }()
- // Preform binary scan.
- extendedScanResults, cleanup, scanErrors, err := dsc.ScanCommand.binaryScan()
- if cleanup != nil {
- defer cleanup()
- }
- if err != nil {
- return
- }
-
- // Print results with docker commands mapping.
- err = xrayutils.NewResultsWriter(extendedScanResults).
- SetOutputFormat(dsc.outputFormat).
- SetIncludeVulnerabilities(dsc.includeVulnerabilities).
- SetIncludeLicenses(dsc.includeLicenses).
- SetPrintExtendedTable(dsc.printExtendedTable).
- SetIsMultipleRootProject(true).
- SetDockerCommandsMapping(dsc.dockerfileCommandsMapping).
- PrintScanResults()
-
- return dsc.ScanCommand.handlePossibleErrors(extendedScanResults.XrayResults, scanErrors, err)
-}
-
-func (dsc *DockerScanCommand) mapDockerLayerToCommand() (err error) {
- log.Debug("Mapping docker layers into commands...")
- resolver, err := dive.GetImageResolver(dive.SourceDockerEngine)
- if err != nil {
- return errorutils.CheckErrorf("failed to map docker layers, is docker running on your machine? error message: %s", err.Error())
- }
- dockerImage, err := resolver.Fetch(dsc.imageTag)
- if err != nil {
- return
- }
- // Create mapping between sha256 hash to dockerfile Command.
- layersMapping := make(map[string]string)
- for _, layer := range dockerImage.Layers {
- layerHash := strings.TrimPrefix(layer.Digest, layerDigestPrefix)
- layersMapping[layerHash] = cleanDockerfileCommand(layer.Command)
- }
- dsc.dockerfileCommandsMapping = layersMapping
- return
-}
-
-// DockerScan command could potentiality have double spaces,
-// Reconstruct the command with only one space between arguments.
-// Example: command from dive: "RUN apt-get install && apt-get install #builtkit".
-// Will resolve to a cleaner command RUN apt-get install && apt-get install
-func cleanDockerfileCommand(rawCommand string) string {
- fields := strings.Fields(rawCommand)
- command := strings.Join(fields, " ")
- return strings.TrimSuffix(command, buildKitSuffix)
+ return dsc.ScanCommand.Run()
 }
 
 // When indexing RPM files inside the docker container, the indexer-app needs to connect to the Xray Server.

xray/commands/scan/scan.go

@@ -5,7 +5,6 @@ import (
  "encoding/json"
  "errors"
  "fmt"
- "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
  "github.com/jfrog/jfrog-cli-core/v2/xray/scangraph"
  xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
  "os/exec"
@@ -16,6 +15,7 @@ import (
  "github.com/jfrog/gofrog/parallel"
  "github.com/jfrog/jfrog-cli-core/v2/common/spec"
  "github.com/jfrog/jfrog-cli-core/v2/utils/config"
+ "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
  "github.com/jfrog/jfrog-cli-core/v2/xray/formats"
  xrutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
  "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns"
@@ -158,96 +158,56 @@ func (scanCmd *ScanCommand) indexFile(filePath string) (*xrayUtils.BinaryGraphNo
 }
 
 func (scanCmd *ScanCommand) Run() (err error) {
- // Preform Binary scan
- extendedScanResults, cleanup, scanErrors, err := scanCmd.binaryScan()
- defer cleanup()
- if err != nil {
- return
- }
- // Print results
- if err = xrutils.NewResultsWriter(extendedScanResults).
- SetOutputFormat(scanCmd.outputFormat).
- SetIncludeVulnerabilities(scanCmd.includeVulnerabilities).
- SetIncludeLicenses(scanCmd.includeLicenses).
- SetPrintExtendedTable(scanCmd.printExtendedTable).
- SetIsMultipleRootProject(true).
- PrintScanResults(); err != nil {
- return
- }
- return scanCmd.handlePossibleErrors(extendedScanResults.XrayResults, scanErrors, err)
-}
-
-// Validate Xray version, download indexer if needed and prepare temp folders
-func (scanCmd *ScanCommand) prepareScanCommand() (xrayVersion string, threads int, cleanup func(), err error) {
+ defer func() {
+ if err != nil {
+ var e *exec.ExitError
+ if errors.As(err, &e) {
+ if e.ExitCode() != coreutils.ExitCodeVulnerableBuild.Code {
+ err = errors.New("Scan command failed. " + err.Error())
+ }
+ }
+ }
+ }()
  xrayManager, xrayVersion, err := xrutils.CreateXrayServiceManagerAndGetVersion(scanCmd.serverDetails)
  if err != nil {
- return
+ return err
  }
 
  // Validate Xray minimum version for graph scan command
  err = clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, scangraph.GraphScanMinXrayVersion)
  if err != nil {
- return
+ return err
  }
 
  if scanCmd.bypassArchiveLimits {
  // Validate Xray minimum version for BypassArchiveLimits flag for indexer
  err = clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, BypassArchiveLimitsMinXrayVersion)
  if err != nil {
- return
+ return err
  }
  }
  log.Info("JFrog Xray version is:", xrayVersion)
  // First download Xray Indexer if needed
  scanCmd.indexerPath, err = DownloadIndexerIfNeeded(xrayManager, xrayVersion)
  if err != nil {
- return
+ return err
  }
  // Create Temp dir for Xray Indexer
  scanCmd.indexerTempDir, err = fileutils.CreateTempDir()
  if err != nil {
- return
- }
- cleanup = func() {
- err = errors.Join(err, fileutils.RemoveTempDir(scanCmd.indexerTempDir))
+ return err
  }
- threads = 1
+ defer func() {
+ e := fileutils.RemoveTempDir(scanCmd.indexerTempDir)
+ if err == nil {
+ err = e
+ }
+ }()
+ threads := 1
  if scanCmd.threads > 1 {
  threads = scanCmd.threads
  }
- return
-}
-
-func (scanCmd *ScanCommand) handlePossibleErrors(flatResults []services.ScanResponse, scanErrors []formats.SimpleJsonError, err error) error {
- // If includeVulnerabilities is false it means that context was provided, so we need to check for build violations.
- // If user provided --fail=false, don't fail the build.
- if scanCmd.fail && !scanCmd.includeVulnerabilities {
- if xrutils.CheckIfFailBuild(flatResults) {
- return xrutils.NewFailBuildError()
- }
- }
- if len(scanErrors) > 0 {
- return errorutils.CheckErrorf(scanErrors[0].ErrorMessage)
- }
-
- if err != nil {
- var e *exec.ExitError
- if errors.As(err, &e) {
- if e.ExitCode() != coreutils.ExitCodeVulnerableBuild.Code {
- err = errors.New("Scan command failed. " + err.Error())
- }
- }
- }
-
- log.Info("Scan completed successfully.")
- return err
-}
 
-func (scanCmd *ScanCommand) binaryScan() (extendedScanResults *xrutils.ExtendedScanResults, cleanup func(), scanErrors []formats.SimpleJsonError, err error) {
- xrayVersion, threads, cleanup, err := scanCmd.prepareScanCommand()
- if err != nil {
- return
- }
  // resultsArr is a two-dimensional array. Each array in it contains a list of ScanResponses that were requested and collected by a specific thread.
  resultsArr := make([][]*services.ScanResponse, threads)
  fileProducerConsumer := parallel.NewRunner(scanCmd.threads, 20000, false)
@@ -269,32 +229,46 @@ func (scanCmd *ScanCommand) binaryScan() (extendedScanResults *xrutils.ExtendedS
  }
  if scanCmd.progress != nil {
  if err = scanCmd.progress.Quit(); err != nil {
- return
+ return err
  }
+
  }
 
  fileCollectingErr := fileCollectingErrorsQueue.GetError()
+ var scanErrors []formats.SimpleJsonError
  if fileCollectingErr != nil {
  scanErrors = append(scanErrors, formats.SimpleJsonError{ErrorMessage: fileCollectingErr.Error()})
  }
  scanErrors = appendErrorSlice(scanErrors, fileProducerErrors)
  scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors)
- extendedScanResults = &xrutils.ExtendedScanResults{XrayResults: flatResults}
+ extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults}
+
+ if err = xrutils.NewResultsWriter(extendedScanResults).
+ SetOutputFormat(scanCmd.outputFormat).
+ SetIncludeVulnerabilities(scanCmd.includeVulnerabilities).
+ SetIncludeLicenses(scanCmd.includeLicenses).
+ SetPrintExtendedTable(scanCmd.printExtendedTable).
+ SetIsMultipleRootProject(true).
+ SetScanType(services.Binary).
+ PrintScanResults(); err != nil {
+ return
+ }
 
+ if err != nil {
+ return err
+ }
  // If includeVulnerabilities is false it means that context was provided, so we need to check for build violations.
  // If user provided --fail=false, don't fail the build.
  if scanCmd.fail && !scanCmd.includeVulnerabilities {
  if xrutils.CheckIfFailBuild(flatResults) {
- err = xrutils.NewFailBuildError()
- return
+ return xrutils.NewFailBuildError()
  }
  }
  if len(scanErrors) > 0 {
- err = errorutils.CheckErrorf(scanErrors[0].ErrorMessage)
- return
+ return errorutils.CheckErrorf(scanErrors[0].ErrorMessage)
  }
  log.Info("Scan completed successfully.")
- return
+ return nil
 }
 
 func NewScanCommand() *ScanCommand {