From 640fbdb3db65a50b063e362bee423f25cdebf5b2 Mon Sep 17 00:00:00 2001 From: Matheus Alcantara Date: Thu, 2 Dec 2021 14:54:19 -0300 Subject: [PATCH] language_detect:chore - improve code readability Signed-off-by: Matheus Alcantara --- .../language_detect/language_detect.go | 105 +++++++++++------- internal/helpers/messages/warn.go | 2 +- 2 files changed, 63 insertions(+), 44 deletions(-) diff --git a/internal/controllers/language_detect/language_detect.go b/internal/controllers/language_detect/language_detect.go index dd96abea5..e4c4ee9b5 100644 --- a/internal/controllers/language_detect/language_detect.go +++ b/internal/controllers/language_detect/language_detect.go @@ -19,12 +19,11 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "github.com/ZupIT/horusec-devkit/pkg/enums/languages" "github.com/ZupIT/horusec-devkit/pkg/utils/logger" - doubleStar "github.com/bmatcuk/doublestar/v4" + "github.com/bmatcuk/doublestar/v4" "github.com/go-enry/go-enry/v2" "github.com/google/uuid" @@ -36,20 +35,25 @@ import ( const prefixGitSubModule = "gitdir: " +// LanguageDetect implements analyzer.LanguageDetect interface, which is +// resposable to detect all languages recursivily to a given base path. type LanguageDetect struct { - configs *config.Config + config *config.Config analysisID uuid.UUID } -func NewLanguageDetect(configs *config.Config, analysisID uuid.UUID) *LanguageDetect { +// NewLanguageDetect create a new language detect. +func NewLanguageDetect(cfg *config.Config, analysisID uuid.UUID) *LanguageDetect { return &LanguageDetect{ analysisID: analysisID, - configs: configs, + config: cfg, } } +// Detect implements analyzer.LanguageDetect.Detect. func (ld *LanguageDetect) Detect(directory string) ([]languages.Language, error) { langs := []string{languages.Leaks.ToString(), languages.Generic.ToString()} + languagesFound, err := ld.getLanguages(directory) if err != nil { logger.LogErrorWithLevel(messages.MsgErrorDetectLanguage, err) @@ -58,80 +62,94 @@ func (ld *LanguageDetect) Detect(directory string) ([]languages.Language, error) langs = ld.appendLanguagesFound(langs, languagesFound) - err = ld.copyProjectToHorusecFolder(directory) + if err := ld.copyProjectToHorusecFolder(directory); err != nil { + return nil, err + } + return ld.filterSupportedLanguages(langs), err } -func (ld *LanguageDetect) getLanguages(directory string) (languagesFound []string, err error) { - filesToSkip, languagesFound, err := ld.walkInPathAndReturnTotalToSkip(directory) - if filesToSkip > 0 { - msg := strings.ReplaceAll(messages.MsgWarnTotalFolderOrFileWasIgnored, "{{0}}", strconv.Itoa(filesToSkip)) - logger.LogWarnWithLevel(msg) +// getLanguages return all unique languages that exists on directory. +func (ld *LanguageDetect) getLanguages(directory string) ([]string, error) { + skipedFiles, languages, err := ld.detectAllLanguages(directory) + if err != nil { + return nil, err + } + + if skipedFiles > 0 { + logger.LogWarnWithLevel(fmt.Sprintf(messages.MsgWarnTotalFolderOrFileWasIgnored, skipedFiles)) } - return ld.uniqueLanguages(languagesFound), err + + return ld.uniqueLanguages(languages), err } -func (ld *LanguageDetect) walkInPathAndReturnTotalToSkip( - directory string) (totalToSkip int, languagesFound []string, err error) { +// detectAllLanguages return all languages that exists on directory and how many +// files was skiped when detecting their languages. +func (ld *LanguageDetect) detectAllLanguages(directory string) (totalToSkip int, languagesFound []string, err error) { err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - currentLanguagesFound, skip := ld.execWalkToGetLanguagesAndReturnIfSkip(path, info) + + newLanguages, skip := ld.detectLanguages(path, info) if skip { totalToSkip++ } - languagesFound = ld.appendLanguagesFound(languagesFound, currentLanguagesFound) + + languagesFound = ld.appendLanguagesFound(languagesFound, newLanguages) return nil }) + return totalToSkip, languagesFound, err } -func (ld *LanguageDetect) execWalkToGetLanguagesAndReturnIfSkip( - path string, info os.FileInfo) (languagesFound []string, skip bool) { - skip = ld.filesAndFoldersToIgnore(path) - if skip { +// detectLanguages return all languages that exists to a given path. If the path should be +// skiped, detectLanguages return nil and true, otherwise will return all languages and false +// if path is not a directory. +func (ld *LanguageDetect) detectLanguages(path string, info os.FileInfo) ([]string, bool) { + if skip := ld.filesAndFoldersToIgnore(path); skip { logger.LogDebugWithLevel(messages.MsgDebugFolderOrFileIgnored, filepath.Clean(path)) + return nil, true } - if !info.IsDir() && !skip { - newLanguages := enry.GetLanguages(path, nil) - logger.LogTraceWithLevel(messages.MsgTraceLanguageFound, - map[string][]string{path: newLanguages}) - languagesFound = append(languagesFound, newLanguages...) + + if info.IsDir() { + return nil, false } - return languagesFound, skip + + languages := enry.GetLanguages(path, nil) + logger.LogTraceWithLevel(messages.MsgTraceLanguageFound, map[string][]string{path: languages}) + return languages, false } func (ld *LanguageDetect) uniqueLanguages(languagesFound []string) (output []string) { for _, language := range languagesFound { if len(output) == 0 { output = append(output, language) - } else { - output = ld.checkIfLanguageExistAndConcat(output, language) + continue } + output = ld.appendIfLanguageNotExists(output, language) } return output } -func (ld *LanguageDetect) checkIfLanguageExistAndConcat(output []string, language string) []string { +func (ld *LanguageDetect) appendIfLanguageNotExists(languages []string, newLanguage string) []string { existing := false - for _, appended := range output { - if appended == language { + for _, lang := range languages { + if lang == newLanguage { existing = true break } } if !existing { - output = append(output, language) + languages = append(languages, newLanguage) } - return output + return languages } func (ld *LanguageDetect) filesAndFoldersToIgnore(path string) bool { - isToSkip := ld.checkDefaultPathsToIgnore(path) || + return ld.checkDefaultPathsToIgnore(path) || ld.checkAdditionalPathsToIgnore(path) || ld.checkFileExtensionInvalid(path) - return isToSkip } func (ld *LanguageDetect) checkDefaultPathsToIgnore(path string) bool { @@ -140,15 +158,15 @@ func (ld *LanguageDetect) checkDefaultPathsToIgnore(path string) bool { return true } } - if !ld.configs.EnableGitHistoryAnalysis { + if !ld.config.EnableGitHistoryAnalysis { return strings.Contains(path, ".git") } return false } func (ld *LanguageDetect) checkAdditionalPathsToIgnore(path string) bool { - for _, value := range ld.configs.FilesOrPathsToIgnore { - matched, _ := doubleStar.Match(strings.TrimSpace(value), path) + for _, value := range ld.config.FilesOrPathsToIgnore { + matched, _ := doublestar.Match(strings.TrimSpace(value), path) if matched { return true } @@ -166,19 +184,20 @@ func (ld *LanguageDetect) checkFileExtensionInvalid(path string) bool { return false } -// nolint:funlen // method is not necessary broken +// nolint:funlen func (ld *LanguageDetect) copyProjectToHorusecFolder(directory string) error { folderDstName := filepath.Join(directory, ".horusec", ld.analysisID.String()) - err := copy.Copy(directory, folderDstName, ld.filesAndFoldersToIgnore) - if err != nil { + if err := copy.Copy(directory, folderDstName, ld.filesAndFoldersToIgnore); err != nil { logger.LogErrorWithLevel(messages.MsgErrorCopyProjectToHorusecAnalysis, err) return err } + fmt.Print("\n") - logger.LogWarnWithLevel(fmt.Sprintf(messages.MsgWarnMonitorTimeoutIn, ld.configs.TimeoutInSecondsAnalysis)) + logger.LogWarnWithLevel(fmt.Sprintf(messages.MsgWarnMonitorTimeoutIn, ld.config.TimeoutInSecondsAnalysis)) fmt.Print("\n") logger.LogWarnWithLevel(messages.MsgWarnDontRemoveHorusecFolder, folderDstName) fmt.Print("\n") + return ld.copyGitFolderWhenIsSubmodule(directory, folderDstName) } @@ -246,7 +265,7 @@ func (ld *LanguageDetect) isBatFileOrShellFile(lang string) bool { // and replace it inside .horusec to run the gitleaks tool without any problems. //nolint:funlen func (ld *LanguageDetect) copyGitFolderWhenIsSubmodule(directory, folderDstName string) error { - if ld.configs.EnableGitHistoryAnalysis { + if ld.config.EnableGitHistoryAnalysis { isGitSubmodule, originalFolderPath := ld.returnGitFolderOriginalIfIsSubmodule(filepath.Join(directory, ".git")) if isGitSubmodule { logger.LogErrorWithLevel( diff --git a/internal/helpers/messages/warn.go b/internal/helpers/messages/warn.go index d7f40c27b..23f93c46a 100644 --- a/internal/helpers/messages/warn.go +++ b/internal/helpers/messages/warn.go @@ -23,7 +23,7 @@ const ( " Don’t worry, we’ll remove it after the analysis ends automatically! Project sent to folder in location: " MsgWarnBanditFoundInformative = "{HORUSEC_CLI} CAUTION! In your project was found " + "{{0}} details of type informative" - MsgWarnTotalFolderOrFileWasIgnored = "{HORUSEC_CLI} When starting the analysis WE SKIP A TOTAL OF {{0}} FILES " + + MsgWarnTotalFolderOrFileWasIgnored = "{HORUSEC_CLI} When starting the analysis WE SKIP A TOTAL OF %d FILES " + "that are not considered to be analyzed. To see more details use flag --log-level=debug" MsgWarnGitHistoryEnable = "{HORUSEC_CLI} Starting the analysis with git history enabled. " + "ATTENTION the waiting time can be longer when this option is enabled!"