Skip to content

Commit

Permalink
language_detect:chore - improve code readability
Browse files Browse the repository at this point in the history
Signed-off-by: Matheus Alcantara <matheus.alcantara@zup.com.br>
  • Loading branch information
matheusalcantarazup committed Dec 2, 2021
1 parent 59c327f commit 640fbdb
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 44 deletions.
105 changes: 62 additions & 43 deletions internal/controllers/language_detect/language_detect.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,11 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"

"github.com/ZupIT/horusec-devkit/pkg/enums/languages"
"github.com/ZupIT/horusec-devkit/pkg/utils/logger"
doubleStar "github.com/bmatcuk/doublestar/v4"
"github.com/bmatcuk/doublestar/v4"
"github.com/go-enry/go-enry/v2"
"github.com/google/uuid"

Expand All @@ -36,20 +35,25 @@ import (

const prefixGitSubModule = "gitdir: "

// LanguageDetect implements analyzer.LanguageDetect interface, which is
// resposable to detect all languages recursivily to a given base path.
type LanguageDetect struct {
configs *config.Config
config *config.Config
analysisID uuid.UUID
}

func NewLanguageDetect(configs *config.Config, analysisID uuid.UUID) *LanguageDetect {
// NewLanguageDetect create a new language detect.
func NewLanguageDetect(cfg *config.Config, analysisID uuid.UUID) *LanguageDetect {
return &LanguageDetect{
analysisID: analysisID,
configs: configs,
config: cfg,
}
}

// Detect implements analyzer.LanguageDetect.Detect.
func (ld *LanguageDetect) Detect(directory string) ([]languages.Language, error) {
langs := []string{languages.Leaks.ToString(), languages.Generic.ToString()}

languagesFound, err := ld.getLanguages(directory)
if err != nil {
logger.LogErrorWithLevel(messages.MsgErrorDetectLanguage, err)
Expand All @@ -58,80 +62,94 @@ func (ld *LanguageDetect) Detect(directory string) ([]languages.Language, error)

langs = ld.appendLanguagesFound(langs, languagesFound)

err = ld.copyProjectToHorusecFolder(directory)
if err := ld.copyProjectToHorusecFolder(directory); err != nil {
return nil, err
}

return ld.filterSupportedLanguages(langs), err
}

func (ld *LanguageDetect) getLanguages(directory string) (languagesFound []string, err error) {
filesToSkip, languagesFound, err := ld.walkInPathAndReturnTotalToSkip(directory)
if filesToSkip > 0 {
msg := strings.ReplaceAll(messages.MsgWarnTotalFolderOrFileWasIgnored, "{{0}}", strconv.Itoa(filesToSkip))
logger.LogWarnWithLevel(msg)
// getLanguages return all unique languages that exists on directory.
func (ld *LanguageDetect) getLanguages(directory string) ([]string, error) {
skipedFiles, languages, err := ld.detectAllLanguages(directory)
if err != nil {
return nil, err
}

if skipedFiles > 0 {
logger.LogWarnWithLevel(fmt.Sprintf(messages.MsgWarnTotalFolderOrFileWasIgnored, skipedFiles))
}
return ld.uniqueLanguages(languagesFound), err

return ld.uniqueLanguages(languages), err
}

func (ld *LanguageDetect) walkInPathAndReturnTotalToSkip(
directory string) (totalToSkip int, languagesFound []string, err error) {
// detectAllLanguages return all languages that exists on directory and how many
// files was skiped when detecting their languages.
func (ld *LanguageDetect) detectAllLanguages(directory string) (totalToSkip int, languagesFound []string, err error) {
err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
currentLanguagesFound, skip := ld.execWalkToGetLanguagesAndReturnIfSkip(path, info)

newLanguages, skip := ld.detectLanguages(path, info)
if skip {
totalToSkip++
}
languagesFound = ld.appendLanguagesFound(languagesFound, currentLanguagesFound)

languagesFound = ld.appendLanguagesFound(languagesFound, newLanguages)
return nil
})

return totalToSkip, languagesFound, err
}

func (ld *LanguageDetect) execWalkToGetLanguagesAndReturnIfSkip(
path string, info os.FileInfo) (languagesFound []string, skip bool) {
skip = ld.filesAndFoldersToIgnore(path)
if skip {
// detectLanguages return all languages that exists to a given path. If the path should be
// skiped, detectLanguages return nil and true, otherwise will return all languages and false
// if path is not a directory.
func (ld *LanguageDetect) detectLanguages(path string, info os.FileInfo) ([]string, bool) {
if skip := ld.filesAndFoldersToIgnore(path); skip {
logger.LogDebugWithLevel(messages.MsgDebugFolderOrFileIgnored, filepath.Clean(path))
return nil, true
}
if !info.IsDir() && !skip {
newLanguages := enry.GetLanguages(path, nil)
logger.LogTraceWithLevel(messages.MsgTraceLanguageFound,
map[string][]string{path: newLanguages})
languagesFound = append(languagesFound, newLanguages...)

if info.IsDir() {
return nil, false
}
return languagesFound, skip

languages := enry.GetLanguages(path, nil)
logger.LogTraceWithLevel(messages.MsgTraceLanguageFound, map[string][]string{path: languages})
return languages, false
}

func (ld *LanguageDetect) uniqueLanguages(languagesFound []string) (output []string) {
for _, language := range languagesFound {
if len(output) == 0 {
output = append(output, language)
} else {
output = ld.checkIfLanguageExistAndConcat(output, language)
continue
}
output = ld.appendIfLanguageNotExists(output, language)
}
return output
}

func (ld *LanguageDetect) checkIfLanguageExistAndConcat(output []string, language string) []string {
func (ld *LanguageDetect) appendIfLanguageNotExists(languages []string, newLanguage string) []string {
existing := false
for _, appended := range output {
if appended == language {
for _, lang := range languages {
if lang == newLanguage {
existing = true
break
}
}
if !existing {
output = append(output, language)
languages = append(languages, newLanguage)
}
return output
return languages
}

func (ld *LanguageDetect) filesAndFoldersToIgnore(path string) bool {
isToSkip := ld.checkDefaultPathsToIgnore(path) ||
return ld.checkDefaultPathsToIgnore(path) ||
ld.checkAdditionalPathsToIgnore(path) ||
ld.checkFileExtensionInvalid(path)
return isToSkip
}

func (ld *LanguageDetect) checkDefaultPathsToIgnore(path string) bool {
Expand All @@ -140,15 +158,15 @@ func (ld *LanguageDetect) checkDefaultPathsToIgnore(path string) bool {
return true
}
}
if !ld.configs.EnableGitHistoryAnalysis {
if !ld.config.EnableGitHistoryAnalysis {
return strings.Contains(path, ".git")
}
return false
}

func (ld *LanguageDetect) checkAdditionalPathsToIgnore(path string) bool {
for _, value := range ld.configs.FilesOrPathsToIgnore {
matched, _ := doubleStar.Match(strings.TrimSpace(value), path)
for _, value := range ld.config.FilesOrPathsToIgnore {
matched, _ := doublestar.Match(strings.TrimSpace(value), path)
if matched {
return true
}
Expand All @@ -166,19 +184,20 @@ func (ld *LanguageDetect) checkFileExtensionInvalid(path string) bool {
return false
}

// nolint:funlen // method is not necessary broken
// nolint:funlen
func (ld *LanguageDetect) copyProjectToHorusecFolder(directory string) error {
folderDstName := filepath.Join(directory, ".horusec", ld.analysisID.String())
err := copy.Copy(directory, folderDstName, ld.filesAndFoldersToIgnore)
if err != nil {
if err := copy.Copy(directory, folderDstName, ld.filesAndFoldersToIgnore); err != nil {
logger.LogErrorWithLevel(messages.MsgErrorCopyProjectToHorusecAnalysis, err)
return err
}

fmt.Print("\n")
logger.LogWarnWithLevel(fmt.Sprintf(messages.MsgWarnMonitorTimeoutIn, ld.configs.TimeoutInSecondsAnalysis))
logger.LogWarnWithLevel(fmt.Sprintf(messages.MsgWarnMonitorTimeoutIn, ld.config.TimeoutInSecondsAnalysis))
fmt.Print("\n")
logger.LogWarnWithLevel(messages.MsgWarnDontRemoveHorusecFolder, folderDstName)
fmt.Print("\n")

return ld.copyGitFolderWhenIsSubmodule(directory, folderDstName)
}

Expand Down Expand Up @@ -246,7 +265,7 @@ func (ld *LanguageDetect) isBatFileOrShellFile(lang string) bool {
// and replace it inside .horusec to run the gitleaks tool without any problems.
//nolint:funlen
func (ld *LanguageDetect) copyGitFolderWhenIsSubmodule(directory, folderDstName string) error {
if ld.configs.EnableGitHistoryAnalysis {
if ld.config.EnableGitHistoryAnalysis {
isGitSubmodule, originalFolderPath := ld.returnGitFolderOriginalIfIsSubmodule(filepath.Join(directory, ".git"))
if isGitSubmodule {
logger.LogErrorWithLevel(
Expand Down
2 changes: 1 addition & 1 deletion internal/helpers/messages/warn.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const (
" Don’t worry, we’ll remove it after the analysis ends automatically! Project sent to folder in location: "
MsgWarnBanditFoundInformative = "{HORUSEC_CLI} CAUTION! In your project was found " +
"{{0}} details of type informative"
MsgWarnTotalFolderOrFileWasIgnored = "{HORUSEC_CLI} When starting the analysis WE SKIP A TOTAL OF {{0}} FILES " +
MsgWarnTotalFolderOrFileWasIgnored = "{HORUSEC_CLI} When starting the analysis WE SKIP A TOTAL OF %d FILES " +
"that are not considered to be analyzed. To see more details use flag --log-level=debug"
MsgWarnGitHistoryEnable = "{HORUSEC_CLI} Starting the analysis with git history enabled. " +
"ATTENTION the waiting time can be longer when this option is enabled!"
Expand Down

0 comments on commit 640fbdb

Please sign in to comment.