diff --git a/cmd/app/start/start.go b/cmd/app/start/start.go index 8cbe9bd25..2bcc71cec 100644 --- a/cmd/app/start/start.go +++ b/cmd/app/start/start.go @@ -356,7 +356,7 @@ func (s *Start) isRunPromptQuestion(cmd *cobra.Command) bool { func (s *Start) executeAnalysisDirectory() (totalVulns int, err error) { if s.analyzer == nil { - s.analyzer = analyzer.NewAnalyzer(s.configs) + s.analyzer = analyzer.New(s.configs) } return s.analyzer.Analyze() diff --git a/internal/controllers/analyzer/analyzer.go b/internal/controllers/analyzer/analyzer.go index 1166af07e..8de0ab958 100644 --- a/internal/controllers/analyzer/analyzer.go +++ b/internal/controllers/analyzer/analyzer.go @@ -16,14 +16,7 @@ package analyzer import ( "fmt" - "io" - "log" - "os" - "os/signal" - "path" - "path/filepath" "strings" - "sync" "time" "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" @@ -34,61 +27,17 @@ import ( "github.com/ZupIT/horusec-devkit/pkg/enums/severities" enumsVulnerability "github.com/ZupIT/horusec-devkit/pkg/enums/vulnerability" "github.com/ZupIT/horusec-devkit/pkg/utils/logger" - "github.com/briandowns/spinner" "github.com/google/uuid" - "github.com/sirupsen/logrus" "github.com/ZupIT/horusec/config" languagedetect "github.com/ZupIT/horusec/internal/controllers/language_detect" "github.com/ZupIT/horusec/internal/controllers/printresults" - "github.com/ZupIT/horusec/internal/enums/images" "github.com/ZupIT/horusec/internal/helpers/messages" "github.com/ZupIT/horusec/internal/services/docker" - dockerClient "github.com/ZupIT/horusec/internal/services/docker/client" - "github.com/ZupIT/horusec/internal/services/formatters" - "github.com/ZupIT/horusec/internal/services/formatters/c/flawfinder" - dotnetcli "github.com/ZupIT/horusec/internal/services/formatters/csharp/dotnet_cli" - "github.com/ZupIT/horusec/internal/services/formatters/csharp/horuseccsharp" - "github.com/ZupIT/horusec/internal/services/formatters/csharp/scs" - "github.com/ZupIT/horusec/internal/services/formatters/dart/horusecdart" - "github.com/ZupIT/horusec/internal/services/formatters/elixir/mixaudit" - "github.com/ZupIT/horusec/internal/services/formatters/elixir/sobelow" - dependencycheck "github.com/ZupIT/horusec/internal/services/formatters/generic/dependency_check" - "github.com/ZupIT/horusec/internal/services/formatters/generic/semgrep" - "github.com/ZupIT/horusec/internal/services/formatters/generic/trivy" - "github.com/ZupIT/horusec/internal/services/formatters/go/gosec" - "github.com/ZupIT/horusec/internal/services/formatters/go/nancy" - "github.com/ZupIT/horusec/internal/services/formatters/hcl/checkov" - "github.com/ZupIT/horusec/internal/services/formatters/hcl/tfsec" - "github.com/ZupIT/horusec/internal/services/formatters/java/horusecjava" - "github.com/ZupIT/horusec/internal/services/formatters/javascript/horusecjavascript" - "github.com/ZupIT/horusec/internal/services/formatters/javascript/npmaudit" - "github.com/ZupIT/horusec/internal/services/formatters/javascript/yarnaudit" - "github.com/ZupIT/horusec/internal/services/formatters/kotlin/horuseckotlin" - "github.com/ZupIT/horusec/internal/services/formatters/leaks/gitleaks" - "github.com/ZupIT/horusec/internal/services/formatters/leaks/horusecleaks" - "github.com/ZupIT/horusec/internal/services/formatters/nginx/horusecnginx" - "github.com/ZupIT/horusec/internal/services/formatters/php/phpcs" - "github.com/ZupIT/horusec/internal/services/formatters/python/bandit" - "github.com/ZupIT/horusec/internal/services/formatters/python/safety" - "github.com/ZupIT/horusec/internal/services/formatters/ruby/brakeman" - "github.com/ZupIT/horusec/internal/services/formatters/ruby/bundler" - "github.com/ZupIT/horusec/internal/services/formatters/shell/shellcheck" - "github.com/ZupIT/horusec/internal/services/formatters/swift/horusecswift" - "github.com/ZupIT/horusec/internal/services/formatters/yaml/horuseckubernetes" - horusecAPI "github.com/ZupIT/horusec/internal/services/horusec_api" + "github.com/ZupIT/horusec/internal/services/docker/client" + horusec_api "github.com/ZupIT/horusec/internal/services/horusec_api" ) -const LoadingDelay = 200 * time.Millisecond - -// detectVulnerabilityFn is a func that detect vulnerabilities on path. -// detectVulnerabilityFn funcs run all in parallel, so a WaitGroup is required -// to synchronize states of running analysis. -// -// detectVulnerabilityFn funcs can also spawn other detectVulnerabilityFn funcs -// just passing the received WaitGroup to underlying funcs. -type detectVulnerabilityFn func(wg *sync.WaitGroup, path string) error - // LanguageDetect is the interface that detect all languages in some directory. type LanguageDetect interface { Detect(directory string) ([]languages.Language, error) @@ -108,64 +57,45 @@ type HorusecService interface { GetAnalysis(uuid.UUID) (*analysis.Analysis, error) } +// Analyzer is responsible to orchestrate the pipeline of an analysis. +// +// Basically, an analysis has the following steps: +// 1 - Detect all languages on project path. +// 2 - Execute all tools to all languages founded. +// 3 - Send analysis to Horusuec Manager if access token is set. +// 4 - Print analysis results. type Analyzer struct { - docker docker.Docker analysis *analysis.Analysis config *config.Config languageDetect LanguageDetect printController PrintResults horusec HorusecService - formatter formatters.IService - loading *spinner.Spinner + runner *runner } -//nolint:funlen -func NewAnalyzer(cfg *config.Config) *Analyzer { - entity := &analysis.Analysis{ +// New create a new analyzer to a given config. +func New(cfg *config.Config) *Analyzer { + analysiss := &analysis.Analysis{ ID: uuid.New(), CreatedAt: time.Now(), Status: enumsAnalysis.Running, } - dockerAPI := docker.New(dockerClient.NewDockerClient(), cfg, entity.ID) + dockerAPI := docker.New(client.NewDockerClient(), cfg, analysiss.ID) return &Analyzer{ - docker: dockerAPI, - analysis: entity, + analysis: analysiss, config: cfg, - languageDetect: languagedetect.NewLanguageDetect(cfg, entity.ID), - printController: printresults.NewPrintResults(entity, cfg), - horusec: horusecAPI.NewHorusecAPIService(cfg), - formatter: formatters.NewFormatterService(entity, dockerAPI, cfg), - loading: newScanLoading(cfg), - } -} - -func (a *Analyzer) Analyze() (totalVulns int, err error) { - a.removeTrashByInterruptProcess() - totalVulns, err = a.runAnalysis() - a.removeHorusecFolder() - return totalVulns, err -} - -func (a *Analyzer) removeTrashByInterruptProcess() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - for range c { - a.removeHorusecFolder() - log.Fatal() - } - }() -} - -func (a *Analyzer) removeHorusecFolder() { - err := os.RemoveAll(filepath.Join(a.config.ProjectPath, ".horusec")) - logger.LogErrorWithLevel(messages.MsgErrorRemoveAnalysisFolder, err) - if !a.config.DisableDocker { - a.docker.DeleteContainersFromAPI() + languageDetect: languagedetect.NewLanguageDetect(cfg, analysiss.ID), + printController: printresults.NewPrintResults(analysiss, cfg), + horusec: horusec_api.NewHorusecAPIService(cfg), + runner: newRunner(cfg, analysiss, dockerAPI), } } -func (a *Analyzer) runAnalysis() (totalVulns int, err error) { +// Analyze start an analysis and return the total of vulnerabilities founded +// and an error if exists. +// +// nolint: funlen +func (a *Analyzer) Analyze() (int, error) { langs, err := a.languageDetect.Detect(a.config.ProjectPath) if err != nil { return 0, err @@ -176,10 +106,14 @@ func (a *Analyzer) runAnalysis() (totalVulns int, err error) { fmt.Println() } - a.startDetectVulnerabilities(langs) + for _, err := range a.runner.run(langs) { + a.setAnalysisError(err) + } + if err = a.sendAnalysis(); err != nil { logger.LogStringAsError(fmt.Sprintf("[HORUSEC] %s", err.Error())) } + return a.startPrintResults() } @@ -225,236 +159,6 @@ func (a *Analyzer) formatAnalysisToSendToAPI() { } } -// startDetectVulnerabilities handle execution of all analysis in parallel -// -// We ignore the funlen and gocyclo lint here because concurrency code is complicated -// nolint:funlen,gocyclo -func (a *Analyzer) startDetectVulnerabilities(langs []languages.Language) { - var wg sync.WaitGroup - done := make(chan struct{}) - - funcs := a.detectVulnerabilityFuncs() - - a.loading.Start() - - go func() { - defer close(done) - for _, language := range langs { - for _, subPath := range a.config.WorkDir.PathsOfLanguage(language) { - projectSubPath := subPath - a.logProjectSubPath(language, projectSubPath) - - if fn, exist := funcs[language]; exist { - wg.Add(1) - go func() { - defer wg.Done() - if err := fn(&wg, projectSubPath); err != nil { - a.setAnalysisError(err) - } - }() - } - } - } - wg.Wait() - }() - - timeout := time.After(time.Duration(a.config.TimeoutInSecondsAnalysis) * time.Second) - for { - select { - case <-done: - a.loading.Stop() - return - case <-timeout: - a.docker.DeleteContainersFromAPI() - a.config.IsTimeout = true - a.loading.Stop() - return - } - } -} - -// detectVulnerabilityFuncs returns a map of language and functions -// that detect vulnerabilities on some path. -// -// All Languages is greater than 15 -//nolint:funlen -func (a *Analyzer) detectVulnerabilityFuncs() map[languages.Language]detectVulnerabilityFn { - return map[languages.Language]detectVulnerabilityFn{ - languages.CSharp: a.detectVulnerabilityCsharp, - languages.Leaks: a.detectVulnerabilityLeaks, - languages.Go: a.detectVulnerabilityGo, - languages.Java: a.detectVulnerabilityJava, - languages.Kotlin: a.detectVulnerabilityKotlin, - languages.Javascript: a.detectVulnerabilityJavascript, - languages.Python: a.detectVulnerabilityPython, - languages.Ruby: a.detectVulnerabilityRuby, - languages.HCL: a.detectVulnerabilityHCL, - languages.Generic: a.detectVulnerabilityGeneric, - languages.Yaml: a.detectVulnerabilityYaml, - languages.C: a.detectVulnerabilityC, - languages.PHP: a.detectVulnerabilityPHP, - languages.Dart: a.detectVulnerabilityDart, - languages.Elixir: a.detectVulnerabilityElixir, - languages.Shell: a.detectVulnerabilityShell, - languages.Nginx: a.detectVulnerabilityNginx, - languages.Swift: a.detectVulneravilitySwift, - } -} - -func (a *Analyzer) detectVulneravilitySwift(_ *sync.WaitGroup, projectSubPath string) error { - horusecswift.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityCsharp(wg *sync.WaitGroup, projectSubPath string) error { - spawn(wg, horuseccsharp.NewFormatter(a.formatter), projectSubPath) - - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.CSharp)); err != nil { - return err - } - - spawn(wg, scs.NewFormatter(a.formatter), projectSubPath) - dotnetcli.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityLeaks(wg *sync.WaitGroup, projectSubPath string) error { - spawn(wg, horusecleaks.NewFormatter(a.formatter), projectSubPath) - - if a.config.EnableGitHistoryAnalysis { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Leaks)); err != nil { - return err - } - gitleaks.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - } - - return nil -} - -func (a *Analyzer) detectVulnerabilityGo(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Go)); err != nil { - return err - } - - spawn(wg, gosec.NewFormatter(a.formatter), projectSubPath) - nancy.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityJava(_ *sync.WaitGroup, projectSubPath string) error { - horusecjava.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityKotlin(_ *sync.WaitGroup, projectSubPath string) error { - horuseckotlin.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityNginx(_ *sync.WaitGroup, projectSubPath string) error { - horusecnginx.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityJavascript(wg *sync.WaitGroup, projectSubPath string) error { - spawn(wg, horusecjavascript.NewFormatter(a.formatter), projectSubPath) - - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Javascript)); err != nil { - return err - } - spawn(wg, yarnaudit.NewFormatter(a.formatter), projectSubPath) - npmaudit.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityPython(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Python)); err != nil { - return err - } - spawn(wg, bandit.NewFormatter(a.formatter), projectSubPath) - safety.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityRuby(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Ruby)); err != nil { - return err - } - spawn(wg, brakeman.NewFormatter(a.formatter), projectSubPath) - bundler.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityHCL(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.HCL)); err != nil { - return err - } - spawn(wg, tfsec.NewFormatter(a.formatter), projectSubPath) - checkov.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityYaml(_ *sync.WaitGroup, projectSubPath string) error { - horuseckubernetes.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityC(_ *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.C)); err != nil { - return err - } - flawfinder.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityPHP(_ *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.PHP)); err != nil { - return err - } - phpcs.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityGeneric(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Generic)); err != nil { - return err - } - - spawn(wg, trivy.NewFormatter(a.formatter), projectSubPath) - spawn(wg, semgrep.NewFormatter(a.formatter), projectSubPath) - dependencycheck.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityDart(_ *sync.WaitGroup, projectSubPath string) error { - horusecdart.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityElixir(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Elixir)); err != nil { - return err - } - spawn(wg, mixaudit.NewFormatter(a.formatter), projectSubPath) - sobelow.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityShell(_ *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Shell)); err != nil { - return err - } - shellcheck.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) logProjectSubPath(language languages.Language, subPath string) { - if subPath != "" { - msg := fmt.Sprintf("Running %s in subpath: %s", language.ToString(), subPath) - logger.LogDebugWithLevel(msg) - } -} - // nolint:gocyclo func (a *Analyzer) checkIfNoExistHashAndLog(list []string) { for _, hash := range list { @@ -493,15 +197,6 @@ func (a *Analyzer) setAnalysisError(err error) { } } -func (a *Analyzer) getCustomOrDefaultImage(language languages.Language) string { - // Images can be set to empty on config file, so we need to use only if its not empty. - // If its empty we return the default value. - if customImage := a.config.CustomImages[language]; customImage != "" { - return customImage - } - return path.Join(images.DefaultRegistry, images.MapValues()[language]) -} - // SetFalsePositivesAndRiskAcceptInVulnerabilities set analysis vulnerabilities to false // positive or risk accept if the hash exists on falsePositive and riskAccept params. // @@ -565,11 +260,14 @@ func (a *Analyzer) sortVulnerabilitiesByCriticality() *analysis.Analysis { func (a *Analyzer) sortVulnerabilitiesByType() *analysis.Analysis { analysisVulnerabilities := a.getVulnerabilitiesByType(enumsVulnerability.Vulnerability) analysisVulnerabilities = append(analysisVulnerabilities, - a.getVulnerabilitiesByType(enumsVulnerability.RiskAccepted)...) + a.getVulnerabilitiesByType(enumsVulnerability.RiskAccepted)..., + ) analysisVulnerabilities = append(analysisVulnerabilities, - a.getVulnerabilitiesByType(enumsVulnerability.FalsePositive)...) + a.getVulnerabilitiesByType(enumsVulnerability.FalsePositive)..., + ) analysisVulnerabilities = append(analysisVulnerabilities, - a.getVulnerabilitiesByType(enumsVulnerability.Corrected)...) + a.getVulnerabilitiesByType(enumsVulnerability.Corrected)..., + ) a.analysis.AnalysisVulnerabilities = analysisVulnerabilities return a.analysis } @@ -672,22 +370,3 @@ func (a *Analyzer) removeVulnerabilitiesByTypes() *analysis.Analysis { return a.analysis } - -func spawn(wg *sync.WaitGroup, f formatters.IFormatter, src string) { - wg.Add(1) - go func() { - defer wg.Done() - f.StartAnalysis(src) - }() -} - -func newScanLoading(cfg *config.Config) *spinner.Spinner { - loading := spinner.New(spinner.CharSets[11], LoadingDelay) - loading.Suffix = messages.MsgInfoAnalysisLoading - - if cfg.LogLevel == logrus.DebugLevel.String() || cfg.LogLevel == logrus.TraceLevel.String() { - loading.Writer = io.Discard - } - - return loading -} diff --git a/internal/controllers/analyzer/analyzer_test.go b/internal/controllers/analyzer/analyzer_test.go index 2ceeb8e94..566a61bf8 100644 --- a/internal/controllers/analyzer/analyzer_test.go +++ b/internal/controllers/analyzer/analyzer_test.go @@ -25,7 +25,7 @@ import ( "os" "testing" - entitiesAnalysis "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" + "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" "github.com/ZupIT/horusec-devkit/pkg/entities/cli" "github.com/ZupIT/horusec-devkit/pkg/entities/vulnerability" "github.com/ZupIT/horusec-devkit/pkg/enums/languages" @@ -41,7 +41,6 @@ import ( "github.com/ZupIT/horusec/config" "github.com/ZupIT/horusec/internal/entities/workdir" "github.com/ZupIT/horusec/internal/services/docker" - "github.com/ZupIT/horusec/internal/services/formatters" "github.com/ZupIT/horusec/internal/utils/testutil" vulnhash "github.com/ZupIT/horusec/internal/utils/vuln_hash" ) @@ -56,7 +55,7 @@ func BenchmarkAnalyzerAnalyze(b *testing.B) { cfg := config.New() cfg.ProjectPath = testutil.GoExample - analyzer := NewAnalyzer(cfg) + analyzer := New(cfg) for i := 0; i < b.N; i++ { if _, err := analyzer.Analyze(); err != nil { @@ -110,10 +109,10 @@ func TestAnalyzerSetFalsePositivesAndRiskAcceptInVulnerabilities(t *testing.T) { for _, tt := range testcases { t.Run(tt.name, func(t *testing.T) { - analyzer := NewAnalyzer(config.New()) + analyzer := New(config.New()) analyzer.analysis.AnalysisVulnerabilities = append( - analyzer.analysis.AnalysisVulnerabilities, entitiesAnalysis.AnalysisVulnerabilities{ + analyzer.analysis.AnalysisVulnerabilities, analysis.AnalysisVulnerabilities{ AnalysisID: uuid.New(), Vulnerability: tt.vulnerability, }, @@ -144,7 +143,7 @@ func TestAnalyzerSetFalsePositivesAndRiskAcceptInVulnerabilities(t *testing.T) { func TestNewAnalyzer(t *testing.T) { t.Run("Should return type os struct correctly", func(t *testing.T) { - assert.IsType(t, &Analyzer{}, NewAnalyzer(&config.Config{})) + assert.IsType(t, &Analyzer{}, New(&config.Config{})) }) } @@ -153,7 +152,7 @@ func TestAnalyzerWithoutMock(t *testing.T) { cfg := config.New() cfg.ProjectPath = testutil.GoExample - controller := NewAnalyzer(cfg) + controller := New(cfg) _, err := controller.Analyze() assert.NoError(t, err) }) @@ -181,7 +180,7 @@ func TestAnalyzerWithoutMock(t *testing.T) { cfg.HorusecAPIUri = svr.URL defer svr.Close() - controller := NewAnalyzer(cfg) + controller := New(cfg) _, err := controller.Analyze() assert.NoError(t, err) }) @@ -218,7 +217,7 @@ func TestAnalyze(t *testing.T) { horusecAPIMock := testutil.NewHorusecAPIMock() horusecAPIMock.On("SendAnalysis").Return(nil) - horusecAPIMock.On("GetAnalysis").Return(&entitiesAnalysis.Analysis{}, nil) + horusecAPIMock.On("GetAnalysis").Return(&analysis.Analysis{}, nil) dockerMocker := testutil.NewDockerClientMock() dockerMocker.On("CreateLanguageAnalysisContainer").Return("", nil) @@ -231,19 +230,15 @@ func TestAnalyze(t *testing.T) { dockerMocker.On("ContainerRemove").Return(nil) dockerMocker.On("ContainerList").Return([]types.Container{{ID: "test"}}, nil) - dockerSDK := docker.New(dockerMocker, configs, uuid.New()) - controller := &Analyzer{ - docker: dockerSDK, config: configs, languageDetect: languageDetectMock, printController: printResultMock, horusec: horusecAPIMock, - formatter: formatters.NewFormatterService(&entitiesAnalysis.Analysis{}, dockerSDK, configs), - loading: newScanLoading(configs), + runner: newRunner(configs, new(analysis.Analysis), docker.New(dockerMocker, configs, uuid.New())), } - controller.analysis = &entitiesAnalysis.Analysis{ID: uuid.New()} + controller.analysis = &analysis.Analysis{ID: uuid.New()} totalVulns, err := controller.Analyze() assert.NoError(t, err) assert.Equal(t, 0, totalVulns) @@ -289,19 +284,15 @@ func TestAnalyze(t *testing.T) { dockerMocker.On("ContainerRemove").Return(nil) dockerMocker.On("ContainerList").Return([]types.Container{{ID: "test"}}, nil) - dockerSDK := docker.New(dockerMocker, configs, uuid.New()) - controller := &Analyzer{ - docker: dockerSDK, config: configs, languageDetect: languageDetectMock, printController: printResultMock, horusec: horusecAPIMock, - formatter: formatters.NewFormatterService(&entitiesAnalysis.Analysis{}, dockerSDK, configs), - loading: newScanLoading(configs), + runner: newRunner(configs, new(analysis.Analysis), docker.New(dockerMocker, configs, uuid.New())), } - controller.analysis = &entitiesAnalysis.Analysis{ID: uuid.New()} + controller.analysis = &analysis.Analysis{ID: uuid.New()} totalVulns, err := controller.Analyze() assert.NoError(t, err) assert.Equal(t, 0, totalVulns) @@ -319,7 +310,7 @@ func TestAnalyze(t *testing.T) { horusecAPIMock := testutil.NewHorusecAPIMock() horusecAPIMock.On("SendAnalysis").Return(nil) - horusecAPIMock.On("GetAnalysis").Return(&entitiesAnalysis.Analysis{}, nil) + horusecAPIMock.On("GetAnalysis").Return(&analysis.Analysis{}, nil) dockerMocker := testutil.NewDockerClientMock() dockerMocker.On("CreateLanguageAnalysisContainer").Return("", nil) @@ -332,19 +323,15 @@ func TestAnalyze(t *testing.T) { dockerMocker.On("ContainerRemove").Return(nil) dockerMocker.On("ContainerList").Return([]types.Container{{ID: "test"}}, nil) - dockerSDK := docker.New(dockerMocker, configs, uuid.New()) - controller := &Analyzer{ - docker: dockerSDK, config: configs, languageDetect: languageDetectMock, printController: printResultMock, horusec: horusecAPIMock, - formatter: formatters.NewFormatterService(&entitiesAnalysis.Analysis{}, dockerSDK, configs), - loading: newScanLoading(configs), + runner: newRunner(configs, new(analysis.Analysis), docker.New(dockerMocker, configs, uuid.New())), } - controller.analysis = &entitiesAnalysis.Analysis{ID: uuid.New()} + controller.analysis = &analysis.Analysis{ID: uuid.New()} totalVulns, err := controller.Analyze() assert.Error(t, err) assert.Equal(t, 0, totalVulns) @@ -359,13 +346,11 @@ func TestAnalyze(t *testing.T) { horusecAPI := testutil.NewHorusecAPIMock() horusecAPI.On("SendAnalysis").Return(nil) - horusecAPI.On("GetAnalysis").Return(new(entitiesAnalysis.Analysis), nil) - - docker := docker.New(testutil.NewDockerClientMock(), cfg, uuid.New()) + horusecAPI.On("GetAnalysis").Return(new(analysis.Analysis), nil) - analysis := new(entitiesAnalysis.Analysis) - analysis.AnalysisVulnerabilities = append( - analysis.AnalysisVulnerabilities, entitiesAnalysis.AnalysisVulnerabilities{ + analysiss := new(analysis.Analysis) + analysiss.AnalysisVulnerabilities = append( + analysiss.AnalysisVulnerabilities, analysis.AnalysisVulnerabilities{ Vulnerability: vulnerability.Vulnerability{ Severity: severities.Info, }, @@ -377,19 +362,17 @@ func TestAnalyze(t *testing.T) { pr.On("SetAnalysis") analyzer := &Analyzer{ - docker: docker, config: cfg, languageDetect: ld, printController: pr, horusec: horusecAPI, - formatter: formatters.NewFormatterService(analysis, docker, cfg), - loading: newScanLoading(cfg), - analysis: analysis, + analysis: analysiss, + runner: newRunner(cfg, analysiss, docker.New(testutil.NewDockerClientMock(), cfg, uuid.New())), } _, err := analyzer.Analyze() require.NoError(t, err, "Expected no error to execute analysis") - assert.Len(t, analysis.AnalysisVulnerabilities, 1, "Expected that analysis contains info vulnerabilities") + assert.Len(t, analysiss.AnalysisVulnerabilities, 1, "Expected that analysis contains info vulnerabilities") }) } diff --git a/internal/controllers/analyzer/runner.go b/internal/controllers/analyzer/runner.go new file mode 100644 index 000000000..10043eb95 --- /dev/null +++ b/internal/controllers/analyzer/runner.go @@ -0,0 +1,385 @@ +// Copyright 2020 ZUP IT SERVICOS EM TECNOLOGIA E INOVACAO SA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analyzer + +import ( + "fmt" + "io" + "os" + "os/signal" + "path" + "path/filepath" + "sync" + "time" + + "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" + "github.com/ZupIT/horusec-devkit/pkg/enums/languages" + "github.com/ZupIT/horusec-devkit/pkg/utils/logger" + "github.com/briandowns/spinner" + "github.com/sirupsen/logrus" + + "github.com/ZupIT/horusec/config" + "github.com/ZupIT/horusec/internal/enums/images" + "github.com/ZupIT/horusec/internal/helpers/messages" + "github.com/ZupIT/horusec/internal/services/docker" + "github.com/ZupIT/horusec/internal/services/formatters" + "github.com/ZupIT/horusec/internal/services/formatters/c/flawfinder" + dotnetcli "github.com/ZupIT/horusec/internal/services/formatters/csharp/dotnet_cli" + "github.com/ZupIT/horusec/internal/services/formatters/csharp/horuseccsharp" + "github.com/ZupIT/horusec/internal/services/formatters/csharp/scs" + "github.com/ZupIT/horusec/internal/services/formatters/dart/horusecdart" + "github.com/ZupIT/horusec/internal/services/formatters/elixir/mixaudit" + "github.com/ZupIT/horusec/internal/services/formatters/elixir/sobelow" + dependencycheck "github.com/ZupIT/horusec/internal/services/formatters/generic/dependency_check" + "github.com/ZupIT/horusec/internal/services/formatters/generic/semgrep" + "github.com/ZupIT/horusec/internal/services/formatters/generic/trivy" + "github.com/ZupIT/horusec/internal/services/formatters/go/gosec" + "github.com/ZupIT/horusec/internal/services/formatters/go/nancy" + "github.com/ZupIT/horusec/internal/services/formatters/hcl/checkov" + "github.com/ZupIT/horusec/internal/services/formatters/hcl/tfsec" + "github.com/ZupIT/horusec/internal/services/formatters/java/horusecjava" + "github.com/ZupIT/horusec/internal/services/formatters/javascript/horusecjavascript" + "github.com/ZupIT/horusec/internal/services/formatters/javascript/npmaudit" + "github.com/ZupIT/horusec/internal/services/formatters/javascript/yarnaudit" + "github.com/ZupIT/horusec/internal/services/formatters/kotlin/horuseckotlin" + "github.com/ZupIT/horusec/internal/services/formatters/leaks/gitleaks" + "github.com/ZupIT/horusec/internal/services/formatters/leaks/horusecleaks" + "github.com/ZupIT/horusec/internal/services/formatters/nginx/horusecnginx" + "github.com/ZupIT/horusec/internal/services/formatters/php/phpcs" + "github.com/ZupIT/horusec/internal/services/formatters/python/bandit" + "github.com/ZupIT/horusec/internal/services/formatters/python/safety" + "github.com/ZupIT/horusec/internal/services/formatters/ruby/brakeman" + "github.com/ZupIT/horusec/internal/services/formatters/ruby/bundler" + "github.com/ZupIT/horusec/internal/services/formatters/shell/shellcheck" + "github.com/ZupIT/horusec/internal/services/formatters/swift/horusecswift" + "github.com/ZupIT/horusec/internal/services/formatters/yaml/horuseckubernetes" +) + +const spinnerLoadingDelay = 200 * time.Millisecond + +// detectVulnerabilityFn is a func that detect vulnerabilities on path. +// detectVulnerabilityFn funcs run all in parallel, so a WaitGroup is required +// to synchronize states of running analysis. +// +// detectVulnerabilityFn funcs can also spawn other detectVulnerabilityFn funcs +// just passing the received WaitGroup to underlying funcs. +// +// Note that the argument path is a work dir path and not the project path, so this +// value can be empty. +type detectVulnerabilityFn func(wg *sync.WaitGroup, path string) error + +// runner is responsible to orchestrate all executions. +// +// For each language founded on project path, runner will run an analysis using +// the appropriate tool. +type runner struct { + loading *spinner.Spinner + config *config.Config + docker docker.Docker + formatter formatters.IService +} + +func newRunner(cfg *config.Config, analysiss *analysis.Analysis, dockerAPI *docker.API) *runner { + return &runner{ + loading: newScanLoading(cfg), + formatter: formatters.NewFormatterService(analysiss, dockerAPI, cfg), + config: cfg, + docker: dockerAPI, + } +} + +// run handle execution of all analysis in parallel +// +// nolint:funlen,gocyclo +func (r *runner) run(langs []languages.Language) []error { + r.removeTrashByInterruptProcess() + defer r.removeHorusecFolder() + + var ( + wg sync.WaitGroup + errors []error + mutex = new(sync.Mutex) + done = make(chan struct{}) + ) + + funcs := r.detectVulnerabilityFuncs() + + r.loading.Start() + + go func() { + defer close(done) + for _, language := range langs { + for _, subPath := range r.config.WorkDir.PathsOfLanguage(language) { + projectSubPath := subPath + r.logProjectSubPath(language, projectSubPath) + + if fn, exist := funcs[language]; exist { + wg.Add(1) + go func() { + defer wg.Done() + if err := fn(&wg, projectSubPath); err != nil { + mutex.Lock() + errors = append(errors, err) + mutex.Unlock() + } + }() + } + } + } + wg.Wait() + }() + + timeout := time.After(time.Duration(r.config.TimeoutInSecondsAnalysis) * time.Second) + for { + select { + case <-done: + r.loading.Stop() + return errors + case <-timeout: + r.docker.DeleteContainersFromAPI() + r.config.IsTimeout = true + r.loading.Stop() + return errors + } + } +} + +// detectVulnerabilityFuncs returns a map of language and a function +// that detect vulnerabilities on some path. +// +//nolint:funlen +func (r *runner) detectVulnerabilityFuncs() map[languages.Language]detectVulnerabilityFn { + return map[languages.Language]detectVulnerabilityFn{ + languages.CSharp: r.detectVulnerabilityCsharp, + languages.Leaks: r.detectVulnerabilityLeaks, + languages.Go: r.detectVulnerabilityGo, + languages.Java: r.detectVulnerabilityJava, + languages.Kotlin: r.detectVulnerabilityKotlin, + languages.Javascript: r.detectVulnerabilityJavascript, + languages.Python: r.detectVulnerabilityPython, + languages.Ruby: r.detectVulnerabilityRuby, + languages.HCL: r.detectVulnerabilityHCL, + languages.Generic: r.detectVulnerabilityGeneric, + languages.Yaml: r.detectVulnerabilityYaml, + languages.C: r.detectVulnerabilityC, + languages.PHP: r.detectVulnerabilityPHP, + languages.Dart: r.detectVulnerabilityDart, + languages.Elixir: r.detectVulnerabilityElixir, + languages.Shell: r.detectVulnerabilityShell, + languages.Nginx: r.detectVulnerabilityNginx, + languages.Swift: r.detectVulneravilitySwift, + } +} + +func (r *runner) detectVulneravilitySwift(_ *sync.WaitGroup, projectSubPath string) error { + horusecswift.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityCsharp(wg *sync.WaitGroup, projectSubPath string) error { + spawn(wg, horuseccsharp.NewFormatter(r.formatter), projectSubPath) + + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.CSharp)); err != nil { + return err + } + + spawn(wg, scs.NewFormatter(r.formatter), projectSubPath) + dotnetcli.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityLeaks(wg *sync.WaitGroup, projectSubPath string) error { + spawn(wg, horusecleaks.NewFormatter(r.formatter), projectSubPath) + + if r.config.EnableGitHistoryAnalysis { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Leaks)); err != nil { + return err + } + gitleaks.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + } + + return nil +} + +func (r *runner) detectVulnerabilityGo(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Go)); err != nil { + return err + } + + spawn(wg, gosec.NewFormatter(r.formatter), projectSubPath) + nancy.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityJava(_ *sync.WaitGroup, projectSubPath string) error { + horusecjava.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityKotlin(_ *sync.WaitGroup, projectSubPath string) error { + horuseckotlin.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityNginx(_ *sync.WaitGroup, projectSubPath string) error { + horusecnginx.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityJavascript(wg *sync.WaitGroup, projectSubPath string) error { + spawn(wg, horusecjavascript.NewFormatter(r.formatter), projectSubPath) + + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Javascript)); err != nil { + return err + } + spawn(wg, yarnaudit.NewFormatter(r.formatter), projectSubPath) + npmaudit.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityPython(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Python)); err != nil { + return err + } + spawn(wg, bandit.NewFormatter(r.formatter), projectSubPath) + safety.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityRuby(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Ruby)); err != nil { + return err + } + spawn(wg, brakeman.NewFormatter(r.formatter), projectSubPath) + bundler.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityHCL(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.HCL)); err != nil { + return err + } + spawn(wg, tfsec.NewFormatter(r.formatter), projectSubPath) + checkov.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityYaml(_ *sync.WaitGroup, projectSubPath string) error { + horuseckubernetes.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityC(_ *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.C)); err != nil { + return err + } + flawfinder.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityPHP(_ *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.PHP)); err != nil { + return err + } + phpcs.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityGeneric(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Generic)); err != nil { + return err + } + + spawn(wg, trivy.NewFormatter(r.formatter), projectSubPath) + spawn(wg, semgrep.NewFormatter(r.formatter), projectSubPath) + dependencycheck.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityDart(_ *sync.WaitGroup, projectSubPath string) error { + horusecdart.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityElixir(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Elixir)); err != nil { + return err + } + spawn(wg, mixaudit.NewFormatter(r.formatter), projectSubPath) + sobelow.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityShell(_ *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Shell)); err != nil { + return err + } + shellcheck.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) getCustomOrDefaultImage(language languages.Language) string { + // Images can be set to empty on config file, so we need to use only if its not empty. + // If its empty we return the default value. + if customImage := r.config.CustomImages[language]; customImage != "" { + return customImage + } + return path.Join(images.DefaultRegistry, images.MapValues()[language]) +} + +func (r *runner) logProjectSubPath(language languages.Language, subPath string) { + if subPath != "" { + msg := fmt.Sprintf("Running %s in subpath: %s", language.ToString(), subPath) + logger.LogDebugWithLevel(msg) + } +} + +func (r *runner) removeHorusecFolder() { + err := os.RemoveAll(filepath.Join(r.config.ProjectPath, ".horusec")) + logger.LogErrorWithLevel(messages.MsgErrorRemoveAnalysisFolder, err) + if !r.config.DisableDocker { + r.docker.DeleteContainersFromAPI() + } +} + +func (r *runner) removeTrashByInterruptProcess() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for range c { + r.removeHorusecFolder() + os.Exit(1) + } + }() +} + +func spawn(wg *sync.WaitGroup, f formatters.IFormatter, src string) { + wg.Add(1) + go func() { + defer wg.Done() + f.StartAnalysis(src) + }() +} + +func newScanLoading(cfg *config.Config) *spinner.Spinner { + loading := spinner.New(spinner.CharSets[11], spinnerLoadingDelay) + loading.Suffix = messages.MsgInfoAnalysisLoading + + if cfg.LogLevel == logrus.DebugLevel.String() || cfg.LogLevel == logrus.TraceLevel.String() { + loading.Writer = io.Discard + } + + return loading +}