Fixing problem when gitleaks not run (#463)

* Fixing problem when gitleaks not run

* Fix len output

.semver.yaml

@@ -1,4 +1,4 @@
 alpha: 0
 beta: 0
 rc: 0
-release: v2.0.0
+release: v2.0.1

deployments/all-version-cli.txt

@@ -11,3 +11,4 @@ v1-10-2
 v1-10-3
 v2-0-0-rc-1
 v2-0-0
+v2-0-1

deployments/scripts/update-image.sh

@@ -15,13 +15,12 @@
 
 
 UPDATE_TYPE=$1
-SEND_NEW_VERSION_TO_S3=$2
-IS_TO_UPDATE_LATEST=$3
+IS_TO_UPDATE_LATEST=$2
 ACTUAL_RELEASE_FORMATTED=""
 ACTUAL_RELEASE=""
 
 installSemver () {
-    mkdir bin
+    mkdir -p ./bin
     curl -fsSL -o ./bin/install-semver.sh https://raw.githubusercontent.com/ZupIT/horusec-devkit/main/scripts/install-semver.sh
     chmod +x ./bin/install-semver.sh
     ./bin/install-semver.sh
@@ -77,43 +76,43 @@ generateBinaries () {
     ACTUAL_RELEASE_FORMATTED=$(tr '.' '-' <<<"$ACTUAL_RELEASE")
 
     # Build for linux x86
-    if ! env CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -a -installsuffix cgo -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/linux_x86/horusec" ./cmd/app/main.go;
+    if ! env CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -a -installsuffix cgo -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_linux_x86" ./cmd/app/main.go;
     then
         echo "Error when generate Build for linux_x86"
     else
-        echo "1/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/linux_x86/horusec"
+        echo "1/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_linux_x86"
     fi
 
     # Build for linux x64
-    if ! env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/linux_x64/horusec" ./cmd/app/main.go;
+    if ! env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_linux_x64" ./cmd/app/main.go;
     then
         echo "Error when generate Build for linux_x64"
     else
-        echo "2/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/linux_x64/horusec"
+        echo "2/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_linux_x64"
     fi
 
     # Build for Windows x86
-    if ! env GOOS=windows GOARCH=386 go build -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/win_x86/horusec.exe" ./cmd/app/main.go;
+    if ! env GOOS=windows GOARCH=386 go build -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_win_x86.exe" ./cmd/app/main.go;
     then
         echo "Error when generate Build for win_x86"
     else
-        echo "3/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/win_x86/horusec.exe"
+        echo "3/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/win_x86.exe"
     fi
 
     # Build for Windows x64
-    if ! env GOOS=windows GOARCH=amd64 go build -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/win_x64/horusec.exe" ./cmd/app/main.go;
+    if ! env GOOS=windows GOARCH=amd64 go build -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_win_x64.exe" ./cmd/app/main.go;
     then
         echo "Error when generate Build for win_x64"
     else
-        echo "4/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/win_x64/horusec.exe"
+        echo "4/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_win_x64.exe"
     fi
 
     # Build for Mac x64
-    if ! env GOOS=darwin GOARCH=amd64 go build -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/mac_x64/horusec" ./cmd/app/main.go;
+    if ! env GOOS=darwin GOARCH=amd64 go build -o "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_mac_x64" ./cmd/app/main.go;
     then
         echo "Error when generate Build for mac_x64"
     else
-        echo "5/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/mac_x64/horusec"
+        echo "5/5 Binary generated with success in ./bin/horusec/$ACTUAL_RELEASE_FORMATTED/horusec_mac_x64"
     fi
 
     chmod +x "./bin/horusec/$ACTUAL_RELEASE_FORMATTED/linux_x64/horusec"
@@ -123,24 +122,17 @@ generateBinaries () {
 
     sed -i -e "s/$ACTUAL_RELEASE/{{VERSION_NOT_FOUND}}/g" "./config/config.go"
 
-    if [[ "$SEND_NEW_VERSION_TO_S3" == "true" ]]
-    then
-        aws s3 cp "./bin/horusec/$ACTUAL_RELEASE_FORMATTED" "s3://horusec.io/bin/$ACTUAL_RELEASE_FORMATTED" --recursive
-    fi
     docker build -t "horuszup/horusec-cli:$ACTUAL_RELEASE" -f deployments/Dockerfile .
     docker push "horuszup/horusec-cli:$ACTUAL_RELEASE"
 
     if [[ "$IS_TO_UPDATE_LATEST" == "true" ]]
     then
         echo "$ACTUAL_RELEASE_FORMATTED" > ./deployments/version-cli-latest.txt
-        aws s3 cp ./deployments/version-cli-latest.txt s3://horusec.io/bin/version-cli-latest.txt
-        aws s3 cp "./bin/horusec/$ACTUAL_RELEASE_FORMATTED" "s3://horusec.io/bin/latest" --recursive
         docker build -t "horuszup/horusec-cli:latest" -f deployments/Dockerfile .
         docker push "horuszup/horusec-cli:latest"
     fi
 
     echo "$ACTUAL_RELEASE_FORMATTED" >> ./deployments/all-version-cli.txt
-    aws s3 cp ./deployments/all-version-cli.txt s3://horusec.io/bin/all-version-cli.txt
 }
 
 resetAlphaRcToMaster () {

deployments/scripts/update-language.sh

@@ -138,11 +138,13 @@ validateIsToUpdateLatest() {
     fi
 }
 
-installSemver() {
-    chmod +x ./deployments/scripts/install-semver.sh
-    ./deployments/scripts/install-semver.sh
 
-    if ! semver &>/dev/null;
+installSemver () {
+    mkdir -p ./bin
+    curl -fsSL -o ./bin/install-semver.sh https://raw.githubusercontent.com/ZupIT/horusec-devkit/main/scripts/install-semver.sh
+    chmod +x ./bin/install-semver.sh
+    ./bin/install-semver.sh
+    if ! semver &> /dev/null
     then
         exit 1
     fi

deployments/version-cli-latest.txt

@@ -1 +1 @@
-v2-0-0
+v2-0-1

internal/controllers/language_detect/language_detect.go

@@ -145,6 +145,9 @@ func (ld *LanguageDetect) checkDefaultPathsToIgnore(path string) bool {
 			return true
 		}
 	}
+	if !ld.configs.GetEnableGitHistoryAnalysis() {
+		return strings.Contains(path, file.ReplacePathSeparator("/.git/"))
+	}
 	return false
 }
 

internal/enums/images/images.go

@@ -11,7 +11,7 @@ const (
 	Go              = "horuszup/horusec-go:v1.0.0"
 	HCL             = "horuszup/horusec-hcl:v1.0.0"
 	Javascript      = "horuszup/horusec-js:v1.0.0"
-	Leaks           = "horuszup/horusec-leaks:v1.0.0"
+	Leaks           = "horuszup/horusec-leaks:v1.0.1"
 	PHP             = "horuszup/horusec-php:v1.0.0"
 	Python          = "horuszup/horusec-python:v1.0.0"
 	Ruby            = "horuszup/horusec-ruby:v1.0.2"

internal/enums/toignore/to_ignore.go

@@ -16,7 +16,7 @@ package toignore
 
 func GetDefaultFoldersToIgnore() []string {
 	return []string{"/.horusec/", "/.idea/", "/.vscode/", "/tmp/", "/bin/", "/node_modules/", "/vendor/",
-		"go.mod", "go.sum", "/.git/"}
+		"go.mod", "go.sum"}
 }
 
 func GetDefaultExtensionsToIgnore() []string {

internal/enums/toignore/to_ignore_test.go

@@ -22,7 +22,7 @@ import (
 
 func TestGetDefaultFoldersToIgnore(t *testing.T) {
 	t.Run("should success get 10 default files to ignore", func(t *testing.T) {
-		assert.Equal(t, len(GetDefaultFoldersToIgnore()), 10)
+		assert.Equal(t, len(GetDefaultFoldersToIgnore()), 9)
 	})
 }
 

internal/services/formatters/leaks/deployments/.semver.yaml

@@ -1,4 +1,4 @@
 alpha: 0
 beta: 0
 rc: 0
-release: v1.0.0
+release: v1.0.1

internal/services/formatters/leaks/deployments/Dockerfile

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM zricethezav/gitleaks
+FROM zricethezav/gitleaks:v7.5.0
 
-COPY ./horusec-cli/internal/services/formatters/leaks/deployments/rules.toml /rules/rules.toml
+COPY ./internal/services/formatters/leaks/deployments/rules.toml /rules/rules.toml
 
 RUN apk --no-cache add ca-certificates openssh-client jq
 

internal/services/formatters/leaks/deployments/rules.toml

@@ -1,7 +1,7 @@
 title = "gitleaks config"
 
 [[rules]]
-	description = "AWS Manager ID"
+	description = "AWS Access Key"
 	regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
 	tags = ["key", "AWS"]
 
@@ -36,8 +36,20 @@ title = "gitleaks config"
 	tags = ["client", "Twitter"]
 
 [[rules]]
-	description = "Github"
-	regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}'''
+	description = "Github Personal Access Token"
+	regex = '''ghp_[0-9a-zA-Z]{36}'''
+	tags = ["key", "Github"]
+[[rules]]
+	description = "Github OAuth Access Token"
+	regex = '''gho_[0-9a-zA-Z]{36}'''
+	tags = ["key", "Github"]
+[[rules]]
+	description = "Github App Token"
+	regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
+	tags = ["key", "Github"]
+[[rules]]
+	description = "Github Refresh Token"
+	regex = '''ghr_[0-9a-zA-Z]{76}'''
 	tags = ["key", "Github"]
 
 [[rules]]
@@ -102,7 +114,7 @@ title = "gitleaks config"
 
 [[rules]]
 	description = "Slack Webhook"
-	regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}'''
+	regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
 	tags = ["key", "slack"]
 
 [[rules]]
@@ -124,13 +136,51 @@ title = "gitleaks config"
 	description = "Twilio API key"
 	regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}'''
 	tags = ["key", "twilio"]
+
+[[rules]]
+	description = "Dynatrace ttoken"
+	regex = '''dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64}'''
+	tags = ["key", "Dynatrace"]
+
+[[rules]]
+	description = "Shopify shared secret"
+	regex = '''shpss_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "Shopify access token"
+	regex = '''shpat_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "Shopify custom app access token"
+	regex = '''shpca_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "Shopify private app access token"
+	regex = '''shppa_[a-fA-F0-9]{32}'''
+	tags = ["key", "Shopify"]
+
+[[rules]]
+	description = "PyPI upload token"
+	regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{50,1000}'''
+	tags = ["key", "pypi"]
+
 [allowlist]
 	description = "Allowlisted files"
 	files = ['''^\.?gitleaks.toml$''',
-	'''(.*?)(jpg|gif|doc|pdf|bin)$''',
+	'''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''',
 	'''(go.mod|go.sum)$''']
 
+
 [[rules]]
 	description = "Generic Credential"
 	regex = '''(?i)(dbpasswd|dbuser|dbname|dbhost|api_key|apikey|client_secret|clientsecret|access_key|accesskey|secret_key|secretkey)(.{0,20})?['|"]([0-9a-zA-Z-_\/+!{}/=]{4,120})['|"]'''
-	tags = ["key", "API", "generic"]
+	tags = ["key", "API", "generic"]
+
+
+[[rules]]
+	description = "Hardcoded Credential"
+	regex = '''((?i)(set)?password\s*(.?=.?|\()\s*['|\"]\w+[[:print:]]*['|\"])|((?i)(set)?pass\s*(.?=.?|\()\s*['|\"]\w+[[:print:]]*['|\"])|((?i)(set)?pwd\s*(.?=.?|\()\s*['|\"]\w+[[:print:]]*['|\"])|((?i)(set)?passwd\s*(.?=.?|\()\s*['|\"]\w+[[:print:]]*['|\"])|((?i)(set)?senha\s*(.?=.?|\()\s*['|\"]\w+[[:print:]]*['|\"])|([a-zA-Z]{3,10}://[^/\s:@]{3,20}:[^/\s:@]{3,20}@.{1,100}/?.?)'''
+	tags = ["key", "Hardcoded", "generic"]

internal/services/formatters/leaks/gitleaks/config.go

@@ -17,12 +17,12 @@ package gitleaks
 
 const CMD = `
 		{{WORK_DIR}}
-        touch /tmp/results-ANALYSISID.json
-        gitleaks --config="/rules/rules.toml" --owner-path=. --verbose --pretty --report="/tmp/results-ANALYSISID.json" &> /tmp/errorGitleaks-ANALYSISID
-        if [ $? -eq 2 ]; then
-            echo 'ERROR_RUNNING_GITLEAKS'
-            cat /tmp/errorGitleaks-ANALYSISID
-        else
+		touch /tmp/results-ANALYSISID.json /tmp/error-ANALYSISID.txt
+		gitleaks --config-path="/rules/rules.toml" --path="./" --leaks-exit-code="0" --verbose --report="/tmp/results-ANALYSISID.json" &> /tmp/error-ANALYSISID.txt
+		if [ $? -eq 0 ];
+		then
             jq -j -M -c . /tmp/results-ANALYSISID.json
+        else
+            cat /tmp/error-ANALYSISID.txt
         fi
   `

internal/services/formatters/leaks/gitleaks/formatter.go

@@ -16,6 +16,7 @@ package gitleaks
 
 import (
 	"encoding/json"
+	"errors"
 	"strings"
 
 	vulnhash "github.com/ZupIT/horusec/internal/utils/vuln_hash"
@@ -67,7 +68,7 @@ func (f *Formatter) startGitLeaks(projectSubPath string) error {
 }
 
 func (f *Formatter) formatOutputGitLeaks(output string) error {
-	if output == "" {
+	if output == "" || (len(output) >= 4 && output[:4] == "null") {
 		logger.LogDebugWithLevel(messages.MsgDebugOutputEmpty,
 			map[string]interface{}{"tool": tools.GitLeaks.ToString()})
 		f.setGitLeaksOutPutInHorusecAnalysis([]entities.Issue{})
@@ -86,6 +87,9 @@ func (f *Formatter) formatOutputGitLeaks(output string) error {
 func (f *Formatter) parseOutputToIssues(output string) ([]entities.Issue, error) {
 	var issues []entities.Issue
 	err := json.Unmarshal([]byte(output), &issues)
+	if err != nil && strings.Contains(err.Error(), "invalid character") {
+		err = errors.New(output)
+	}
 	logger.LogErrorWithLevel(f.GetAnalysisIDErrorMessage(tools.GitLeaks, output), err)
 	return issues, err
 }
@@ -100,7 +104,7 @@ func (f *Formatter) setGitLeaksOutPutInHorusecAnalysis(issues []entities.Issue)
 func (f *Formatter) setupVulnerabilitiesSeveritiesGitLeaks(issue *entities.Issue) (
 	vulnerabilitySeverity *vulnerability.Vulnerability) {
 	vulnerabilitySeverity = f.getDefaultSeverity()
-	vulnerabilitySeverity.Severity = severities.Unknown
+	vulnerabilitySeverity.Severity = severities.Critical
 	vulnerabilitySeverity.Details = issue.Rule
 	vulnerabilitySeverity.Code = f.GetCodeWithMaxCharacters(issue.Line, 0)
 	vulnerabilitySeverity.File = issue.File