Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JFrog Advanced Security - Secrets & IaC scans #788

Merged
merged 70 commits into from
Jun 13, 2023
Merged
Show file tree
Hide file tree
Changes from 68 commits
Commits
Show all changes
70 commits
Select commit Hold shift + click to select a range
88c3f7b
applicabilityscanner.go
orto17 Apr 3, 2023
fd8ab72
added app scan results to run audit
orto17 Apr 3, 2023
8a465b4
added applicability result column to table
orto17 Apr 4, 2023
67f0f9d
analyzer manager executer
orto17 Apr 16, 2023
66cdf20
ConvertToApplicableTableValue
orto17 Apr 16, 2023
8b6f0a1
omit empty column implementation, second sort
orto17 Apr 16, 2023
9ebb80b
with violations cves results
orto17 Apr 16, 2023
108e4ea
technologies eligible for applicability scan
orto17 Apr 16, 2023
7d9ed21
with dependency tree, fixed result writer test
orto17 Apr 17, 2023
8bedfb3
unit tests
orto17 Apr 18, 2023
2401d3f
all unit tests for app scanner
orto17 Apr 18, 2023
8b3400f
updates after static analysis
orto17 Apr 18, 2023
9bfdf3a
with real path to analyzer manager, tests fixed
orto17 Apr 18, 2023
c12b402
TestApplicabilityScanManager_ShouldRun_AllConditionsMet
orto17 Apr 18, 2023
41e5884
Merge remote-tracking branch 'upstream/dev' into Applicable-vulnerabi…
orto17 Apr 19, 2023
2c29b86
Smallfixes after review
orto17 Apr 19, 2023
32ea54d
analyzer manager set env variables
orto17 Apr 20, 2023
bb54ab7
handle error code 31 from am
orto17 Apr 20, 2023
d9fe0aa
secret scanner, print table
orto17 Apr 24, 2023
5b735e6
changes after Tal review
orto17 Apr 24, 2023
0ca272d
tech eligible for scan fix
orto17 Apr 24, 2023
fb035ed
secret scanner improvements
orto17 May 1, 2023
182cb41
secret scanner improvements
orto17 May 1, 2023
eb108f8
Merge branch 'Applicable-vulnerability-column' into secrets-scanner
orto17 May 1, 2023
dc67147
secret scanner improvements
orto17 May 1, 2023
3a9d2d6
tryExtractRelativePath
orto17 May 3, 2023
2629f3d
severity column design
orto17 May 3, 2023
6864666
table improvements, tests
orto17 May 3, 2023
1682570
added secrets to non table output
orto17 May 3, 2023
4ffe8bb
iac scanner
orto17 May 8, 2023
9fe4e09
iac scanner
orto17 May 8, 2023
57e4ddc
print iac results
orto17 May 9, 2023
26af01e
updated iac table column names
orto17 May 9, 2023
1acedff
Merge remote-tracking branch 'upstream/dev' into secrets-scanner
orto17 May 10, 2023
e398117
GetSeverity()
orto17 May 14, 2023
9f31d53
Merge branch 'dev' into secrets-scanner
orto17 May 15, 2023
6895b1e
merged dev
orto17 May 15, 2023
fc93f3b
fixed conflicts
orto17 May 17, 2023
9d85cb5
updated scanner according to dev changes
orto17 May 21, 2023
98380d5
refactor iac and secrets structs
orto17 May 21, 2023
5816b5c
go mod tidy
orto17 May 21, 2023
f387563
IacSecretsRow
orto17 May 21, 2023
41bcc18
Merge remote-tracking branch 'upstream/dev' into secrets-scanner
orto17 May 21, 2023
25018ce
merged dev
orto17 May 21, 2023
8e793d9
cmd dir
orto17 May 21, 2023
6dd16a4
fixed config file output field
orto17 May 22, 2023
0feb7a6
fixed tests
orto17 May 22, 2023
b87b830
Merge branch 'dev' into secrets-scanner
orto17 May 28, 2023
2a70cbf
unit tests
orto17 May 28, 2023
c6eec36
Merge branch 'dev' into secrets-scanner
orto17 May 28, 2023
36241e8
empty url warning message
orto17 May 28, 2023
7370225
ApplicabilityScanResults
orto17 May 28, 2023
c41ad00
secret scan failure message
orto17 May 28, 2023
1139820
secret scan failure message
orto17 May 28, 2023
951db93
not entitled const not in use
orto17 May 28, 2023
161bbf1
secrets ExtractRelativePath
orto17 May 29, 2023
9ee4848
ca column logic fix
orto17 May 29, 2023
17e15aa
npm test
orto17 May 30, 2023
6267ef5
level to severity map
orto17 May 31, 2023
7d485ae
errorutils.CheckError(err)
orto17 May 31, 2023
1beb1b6
SkippedDirs
orto17 May 31, 2023
42ade5d
cmd.Process.Kill()
orto17 Jun 4, 2023
87e02bd
fixed exec defer
orto17 Jun 5, 2023
11a26c4
exit code const
orto17 Jun 6, 2023
134e36e
updates after Eyal's review
orto17 Jun 7, 2023
aaafafb
updates after Eyal's review
orto17 Jun 7, 2023
537f7b2
Merge remote-tracking branch 'upstream/dev' into secrets-scanner
orto17 Jun 7, 2023
8fb676d
fixed tests
orto17 Jun 7, 2023
eadb9c2
changes after code review
orto17 Jun 12, 2023
63a4ded
Merge branch 'dev' into secrets-scanner
orto17 Jun 12, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
package audit
package jas

import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"

"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/xray/utils"
Expand All @@ -17,70 +13,55 @@ import (
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
"github.com/owenrumney/go-sarif/v2/sarif"
"gopkg.in/yaml.v2"
"os"
"path/filepath"
"strings"
)

const (
ApplicabilityFeatureId = "contextual_analysis"
applicabilityScanType = "analyze-applicability"
applicabilityScanFailureMessage = "failed to run applicability scan. Cause: %s"
noEntitledExitCode = 31
applicabilityScanCommand = "ca"
)

var (
analyzerManagerExecuter utils.AnalyzerManagerInterface = &utils.AnalyzerManager{}
technologiesEligibleForApplicabilityScan = []coreutils.Technology{coreutils.Npm, coreutils.Pip,
technologiesEligibleForApplicabilityScan = []coreutils.Technology{coreutils.Npm, coreutils.Pip,
coreutils.Poetry, coreutils.Pipenv, coreutils.Pypi}
skippedDirs = []string{"**/*test*/**", "**/*venv*/**", "**/*node_modules*/**", "**/*target*/**"}
)

func GetExtendedScanResults(results []services.ScanResponse, dependencyTrees []*xrayUtils.GraphNode,
serverDetails *config.ServerDetails) (extendedResults *utils.ExtendedScanResults, err error) {
if err = utils.CreateAnalyzerManagerLogDir(); err != nil {
return
}
applicabilityScanManager, cleanupFunc, err := NewApplicabilityScanManager(results, dependencyTrees, serverDetails)
// The getApplicabilityScanResults function runs the applicability scan flow, which includes the following steps:
// Creating an ApplicabilityScanManager object.
// Checking if the scanned project is eligible for applicability scan.
// Running the analyzer manager executable.
// Parsing the analyzer manager results.
// Return values:
// map[string]string: A map containing the applicability result of each XRAY CVE.
// bool: true if the user is entitled to the applicability scan, false otherwise.
// error: An error object (if any).
func getApplicabilityScanResults(results []services.ScanResponse, dependencyTrees []*xrayUtils.GraphNode,
serverDetails *config.ServerDetails, analyzerManager utils.AnalyzerManagerInterface) (map[string]string, bool, error) {
applicabilityScanManager, cleanupFunc, err := newApplicabilityScanManager(results, dependencyTrees, serverDetails, analyzerManager)
if err != nil {
return nil, fmt.Errorf(applicabilityScanFailureMessage, err.Error())
return nil, false, fmt.Errorf(applicabilityScanFailureMessage, err.Error())
}
defer func() {
if cleanupFunc != nil {
e := cleanupFunc()
if err == nil {
err = e
}
cleanupError := cleanupFunc()
err = errors.Join(err, cleanupError)
}
orto17 marked this conversation as resolved.
Show resolved Hide resolved
}()
eligibleForApplicabilityScan, err := applicabilityScanManager.eligibleForApplicabilityScan()
if err != nil {
return nil, fmt.Errorf(applicabilityScanFailureMessage, err.Error())
if !applicabilityScanManager.eligibleForApplicabilityScan() {
log.Debug("The conditions for running the applicability scan are not met. Skipping the execution of the Analyzer Manager")
return nil, false, nil
}
if !eligibleForApplicabilityScan {
if len(serverDetails.Url) == 0 {
log.Warn("To include 'Contextual Analysis' information as part of the audit output, please run the 'jf c add' command before running this command.")
if err = applicabilityScanManager.run(); err != nil {
if utils.IsNotEntitledError(err) || utils.IsUnsupportedCommandError(err) {
return nil, false, nil
}
log.Debug("The conditions required for running 'Contextual Analysis' as part of the audit are not met.")
return &utils.ExtendedScanResults{XrayResults: results, ApplicabilityScannerResults: nil, EntitledForJas: false}, nil
return nil, true, fmt.Errorf(applicabilityScanFailureMessage, err.Error())
}
entitledForJas, err := applicabilityScanManager.Run()
if err != nil {
return nil, fmt.Errorf(applicabilityScanFailureMessage, err.Error())
}
if !entitledForJas {
log.Debug("the current user is not entitled for the Advanced Security package")
return &utils.ExtendedScanResults{XrayResults: results, ApplicabilityScannerResults: nil, EntitledForJas: false}, nil
}
applicabilityScanResults := applicabilityScanManager.getApplicabilityScanResults()
extendedScanResults := utils.ExtendedScanResults{XrayResults: results, ApplicabilityScannerResults: applicabilityScanResults, EntitledForJas: true}
return &extendedScanResults, nil
}

func (a *ApplicabilityScanManager) eligibleForApplicabilityScan() (bool, error) {
analyzerManagerExist, err := a.analyzerManager.ExistLocally()
if err != nil {
return false, err
}
return analyzerManagerExist && resultsIncludeEligibleTechnologies(getXrayVulnerabilities(a.xrayResults),
getXrayViolations(a.xrayResults)) && len(a.serverDetails.Url) > 0, nil
return applicabilityScanManager.applicabilityScanResults, true, nil
}

// Applicability scan is relevant only to specific programming languages (the languages in this list:
Expand All @@ -107,18 +88,18 @@ func resultsIncludeEligibleTechnologies(xrayVulnerabilities []services.Vulnerabi
}

type ApplicabilityScanManager struct {
applicabilityScannerResults map[string]string
xrayResults []services.ScanResponse
xrayDirectVulnerabilities []services.Vulnerability
xrayDirectViolations []services.Violation
configFileName string
resultsFileName string
analyzerManager utils.AnalyzerManagerInterface
serverDetails *config.ServerDetails
applicabilityScanResults map[string]string
xrayVulnerabilities []services.Vulnerability
xrayViolations []services.Violation
xrayResults []services.ScanResponse
configFileName string
resultsFileName string
analyzerManager utils.AnalyzerManagerInterface
serverDetails *config.ServerDetails
}

func NewApplicabilityScanManager(xrayScanResults []services.ScanResponse, dependencyTrees []*xrayUtils.GraphNode,
serverDetails *config.ServerDetails) (manager *ApplicabilityScanManager, cleanup func() error, err error) {
func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, dependencyTrees []*xrayUtils.GraphNode,
serverDetails *config.ServerDetails, analyzerManager utils.AnalyzerManagerInterface) (manager *ApplicabilityScanManager, cleanup func() error, err error) {
directDependencies := getDirectDependenciesList(dependencyTrees)
tempDir, err := fileutils.CreateTempDir()
if err != nil {
Expand All @@ -128,17 +109,21 @@ func NewApplicabilityScanManager(xrayScanResults []services.ScanResponse, depend
return fileutils.RemoveTempDir(tempDir)
}
return &ApplicabilityScanManager{
applicabilityScannerResults: map[string]string{},
xrayDirectVulnerabilities: extractXrayDirectVulnerabilities(xrayScanResults, directDependencies),
xrayDirectViolations: extractXrayDirectViolations(xrayScanResults, directDependencies),
xrayResults: xrayScanResults,
configFileName: filepath.Join(tempDir, "config.yaml"),
resultsFileName: filepath.Join(tempDir, "results.sarif"),
analyzerManager: analyzerManagerExecuter,
serverDetails: serverDetails,
applicabilityScanResults: map[string]string{},
xrayVulnerabilities: extractXrayDirectVulnerabilities(xrayScanResults, directDependencies),
xrayViolations: extractXrayDirectViolations(xrayScanResults, directDependencies),
configFileName: filepath.Join(tempDir, "config.yaml"),
resultsFileName: filepath.Join(tempDir, "results.sarif"),
xrayResults: xrayScanResults,
analyzerManager: analyzerManager,
serverDetails: serverDetails,
}, cleanup, nil
}

func (a *ApplicabilityScanManager) eligibleForApplicabilityScan() bool {
return resultsIncludeEligibleTechnologies(getXrayVulnerabilities(a.xrayResults), getXrayViolations(a.xrayResults))
}

// This function gets a liat of xray scan responses that contains direct and indirect violations, and returns only direct
// violation of the scanned project, ignoring indirect violations
func extractXrayDirectViolations(xrayScanResults []services.ScanResponse, directDependencies []string) []services.Violation {
Expand Down Expand Up @@ -197,33 +182,30 @@ func getXrayViolations(xrayScanResults []services.ScanResponse) []services.Viola
return xrayViolations
}

func (a *ApplicabilityScanManager) getApplicabilityScanResults() map[string]string {
return a.applicabilityScannerResults
}

func (a *ApplicabilityScanManager) Run() (bool, error) {
func (a *ApplicabilityScanManager) run() error {
var err error
defer func() {
if a.deleteApplicabilityScanProcessFiles() != nil {
e := a.deleteApplicabilityScanProcessFiles()
if deleteJasProcessFiles(a.configFileName, a.resultsFileName) != nil {
e := deleteJasProcessFiles(a.configFileName, a.resultsFileName)
if err == nil {
err = e
}
}
orto17 marked this conversation as resolved.
Show resolved Hide resolved
}()
if !a.directDependenciesExist() {
return true, nil
return nil
}
if err = a.createConfigFile(); err != nil {
return true, err
}
if entitledForJas, err := a.runAnalyzerManager(); err != nil {
return entitledForJas, err
return err
}
if err = a.parseResults(); err != nil {
return true, err
if err = a.runAnalyzerManager(); err != nil {
return err
}
return true, nil
return a.parseResults()
}

func (a *ApplicabilityScanManager) directDependenciesExist() bool {
return len(createCveList(a.xrayVulnerabilities, a.xrayViolations)) > 0
}

type applicabilityScanConfig struct {
Expand All @@ -239,16 +221,12 @@ type scanConfiguration struct {
SkippedDirs []string `yaml:"skipped-folders"`
}

func (a *ApplicabilityScanManager) directDependenciesExist() bool {
return len(createCveList(a.xrayDirectVulnerabilities, a.xrayDirectViolations)) > 0
}

func (a *ApplicabilityScanManager) createConfigFile() error {
currentDir, err := coreutils.GetWorkingDirectory()
if err != nil {
return err
}
cveWhiteList := utils.RemoveDuplicateValues(createCveList(a.xrayDirectVulnerabilities, a.xrayDirectViolations))
cveWhiteList := utils.RemoveDuplicateValues(createCveList(a.xrayVulnerabilities, a.xrayViolations))
configFileContent := applicabilityScanConfig{
Scans: []scanConfiguration{
{
Expand All @@ -262,77 +240,43 @@ func (a *ApplicabilityScanManager) createConfigFile() error {
},
}
yamlData, err := yaml.Marshal(&configFileContent)
if err != nil {
if errorutils.CheckError(err) != nil {
return err
}
err = os.WriteFile(a.configFileName, yamlData, 0644)
return err
return errorutils.CheckError(err)
}

// Runs the analyzerManager app and returns a boolean indicates if the user is entitled for
// Runs the analyzerManager app and returns a boolean to indicate whether the user is entitled for
// advance security feature
func (a *ApplicabilityScanManager) runAnalyzerManager() (bool, error) {
func (a *ApplicabilityScanManager) runAnalyzerManager() error {
if err := utils.SetAnalyzerManagerEnvVariables(a.serverDetails); err != nil {
return true, err
}

if err := a.analyzerManager.Exec(a.configFileName); err != nil {
if exitError, ok := err.(*exec.ExitError); ok {
exitCode := exitError.ExitCode()
// User not entitled error
if exitCode == noEntitledExitCode {
return false, err
}
}
return err
}
return true, nil
return a.analyzerManager.Exec(a.configFileName, applicabilityScanCommand)
}

func (a *ApplicabilityScanManager) parseResults() error {
report, err := sarif.Open(a.resultsFileName)
if err != nil {
if errorutils.CheckError(err) != nil {
return err
}
var fullVulnerabilitiesList []*sarif.Result
if len(report.Runs) > 0 {
fullVulnerabilitiesList = report.Runs[0].Results
}

xrayCves := utils.RemoveDuplicateValues(createCveList(a.xrayDirectVulnerabilities, a.xrayDirectViolations))
xrayCves := utils.RemoveDuplicateValues(createCveList(a.xrayVulnerabilities, a.xrayViolations))
for _, xrayCve := range xrayCves {
a.applicabilityScannerResults[xrayCve] = utils.ApplicabilityUndeterminedStringValue
a.applicabilityScanResults[xrayCve] = utils.ApplicabilityUndeterminedStringValue
}

for _, vulnerability := range fullVulnerabilitiesList {
applicableVulnerabilityName := getVulnerabilityName(*vulnerability.RuleID)
if isVulnerabilityApplicable(vulnerability) {
a.applicabilityScannerResults[applicableVulnerabilityName] = utils.ApplicableStringValue
a.applicabilityScanResults[applicableVulnerabilityName] = utils.ApplicableStringValue
} else {
a.applicabilityScannerResults[applicableVulnerabilityName] = utils.NotApplicableStringValue
}
}
return nil
}

func (a *ApplicabilityScanManager) deleteApplicabilityScanProcessFiles() error {
exist, err := fileutils.IsFileExists(a.configFileName, false)
if err != nil {
return err
}
if exist {
err = os.Remove(a.configFileName)
if errorutils.CheckError(err) != nil {
return err
}
}
exist, err = fileutils.IsFileExists(a.resultsFileName, false)
if err != nil {
return err
}
if exist {
err = os.Remove(a.resultsFileName)
if errorutils.CheckError(err) != nil {
return err
a.applicabilityScanResults[applicableVulnerabilityName] = utils.NotApplicableStringValue
}
}
return nil
Expand Down
Loading
Loading