Merge branch 'main' into impl-data-model-gitlab

* main: (76 commits)
  update aws descriptions (trufflesecurity#3529)
  enforce timeout on circleci test (trufflesecurity#3528)
  rm snifftest (trufflesecurity#3527)
  Redact more source credentials (trufflesecurity#3526)
  Create global log redaction capability (trufflesecurity#3522)
  Adding basic "what is trufflehog" to the readme (trufflesecurity#3514)
  Handle custom detector response and include in extra data (trufflesecurity#3411)
  fix: fixed validation logic for `calendarific` (trufflesecurity#3480)
  fix(deps): update github.com/tailscale/depaware digest to 3d7f3b3 (trufflesecurity#3518)
  Move DecoderType into ResultWithMetadata trufflesecurity#3502
  Addeded 403 account block status code handling for gitlab (trufflesecurity#3471)
  updated gcpapplicationdefaultcredentials detector results with RawV2 (trufflesecurity#3499)
  fix(deps): update module github.com/brianvoe/gofakeit/v7 to v7.1.1 (trufflesecurity#3512)
  fix(deps): update module github.com/schollz/progressbar/v3 to v3.17.0 (trufflesecurity#3510)
  fix(deps): update module cloud.google.com/go/secretmanager to v1.14.2 (trufflesecurity#3498)
  Adds a logging section in the contributing guidelines (trufflesecurity#3509)
  fix: fixed verifcation pattern logic for `bulksms` (trufflesecurity#3478)
  Extend `algoliaadminkey` with additional checks (trufflesecurity#3459)
  fix(deps): update module google.golang.org/api to v0.203.0 (trufflesecurity#3497)
  fix: added correct api endpoint for verification & logic for Aeroworkflow (trufflesecurity#3435)
  ...

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,5 +8,4 @@ Explain the purpose of the PR.
 
 ### Checklist:
 * [ ] Tests passing (`make test-community`)?
-* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
-
+* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/welcome/install/#local-installation))?

.github/workflows/smoke.yml

@@ -16,5 +16,38 @@ jobs:
       - name: Smoke
         run: |
           set -e
-          go run . git https://github.com/dustin-decker/secretsandstuff.git
-          go run . github --repo https://github.com/dustin-decker/secretsandstuff.git
+          go run . git https://github.com/dustin-decker/secretsandstuff.git > /dev/null
+          go run . github --repo https://github.com/dustin-decker/secretsandstuff.git > /dev/null
+  zombies:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23"
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Run trufflehog
+        run: |
+          set -e
+          go run . git --no-verification file://. > /dev/null
+          # This case previously had a deadlock issue and left zombies after trufflehog exited #3379
+          go run . git --no-verification https://github.com/git-test-fixtures/binary.git > /dev/null
+      - name: Check for running git processes and zombies
+        run: |
+          if pgrep -x "git" > /dev/null
+          then
+            echo "Git processes are still running"
+            exit 1
+          else
+            echo "No git processes found"
+          fi
+
+          if ps -A -ostat,ppid | grep -e '[zZ]' > /dev/null
+          then
+            echo "Zombie processes found"
+            exit 1
+          else
+            echo "No zombie processes found"
+          fi

.gitignore

@@ -9,3 +9,4 @@ tmp/go-test.json
 .captain/detectors/timings.yaml
 .captain/detectors/quarantines.yaml
 .captain/detectors/flakes.yaml
+.vscode

CONTRIBUTING.md

@@ -11,3 +11,26 @@ Contributors need to [sign our CLA](https://cla-assistant.io/trufflesecurity/tru
 ## Adding new secret detectors
 
 We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
+
+## Logging in TruffleHog
+
+**Use fields over format strings**. For structured logging, fields allow us to better filter and search through logs than embedding data in the message.
+
+**Differentiate logs coming from dependencies**. This can be done with a `"dep"` field that gets passed to the library. Sometimes it’s not possible to do this.
+
+Limit log levels to _**info**_ (indicate normal or expected operation) and _**error**_ (functionality is impeded and should be checked by an engineer)
+
+**Choose an appropriate verbosity level**
+```
+0. — logs we always want to see
+1. — logs we could possibly want to turn off
+2. — logs that are useful for debugging
+3. — frequently called logs that may produce a lot of output
+4. — extremely verbose logs or logs containing sensitive information
+5. — ultimate verbosity
+```
+Example: `Logger().V(2).Info("skipping file: extension is ignored", "ext", mimeExt)`
+
+**Either log an error or return it**. Doing one or the other will help defer logging for when there is more context for it and prevent duplicate “bubbling up” logs.
+
+**Log contextual information**. Every log emitted should contain this context via fields to easily filter and search.

Makefile

@@ -61,8 +61,5 @@ release-protos-image:
 	docker buildx build --push --platform=linux/amd64,linux/arm64 \
 	-t ${PROTOS_IMAGE} -f hack/Dockerfile.protos .
 
-snifftest:
-	./hack/snifftest/snifftest.sh
-
 test-release:
 	goreleaser release --clean --skip-publish --snapshot

README.md

@@ -36,6 +36,26 @@ We take the revenue from the enterprise product to fund more awesome open source
 
 </div>
 
+# What is TruffleHog 🐽
+
+TruffleHog is the most powerful secrets **Discovery, Classification, Validation,** and **Analysis** tool. In this context secret refers to a credential a machine uses to authenticate itself to another machine. This includes API keys, database passwords, private encryption keys, and more...
+
+## Discovery 🔍
+
+TruffleHog can look for secrets in many places including Git, chats, wikis, logs, API testing platforms, object stores, filesystems and more
+
+## Classification 📁
+
+TruffleHog classifies over 800 secret types, mapping them back to the specific identity they belong to. Is it an AWS secret? Stripe secret? Cloudflare secret? Postgres password? SSL Private key? Sometimes its hard to tell looking at it, so TruffleHog classifies everything it finds.
+
+## Validation ✅
+
+For every secret TruffleHog can classify, it can also log in to confirm if that secret is live or not. This step is critical to know if there’s an active present danger or not.
+
+## Analysis 🔬
+
+For the 20 some of the most commonly leaked out credential types, instead of sending one request to check if the secret can log in, TruffleHog can send many requests to learn everything there is to know about the secret. Who created it? What resources can it access? What permissions does it have on those resources?
+
 # :loudspeaker: Join Our Community
 
 Have questions? Feedback? Jump in slack or discord and hang out with us

go.mod

@@ -14,8 +14,8 @@ replace github.com/coinbase/waas-client-library-go => github.com/trufflesecurity
 replace github.com/STARRY-S/zip => github.com/STARRY-S/zip v0.1.0
 
 require (
-	cloud.google.com/go/secretmanager v1.14.1
-	cloud.google.com/go/storage v1.44.0
+	cloud.google.com/go/secretmanager v1.14.2
+	cloud.google.com/go/storage v1.45.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.13
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2
 	github.com/BobuSumisu/aho-corasick v1.0.3
@@ -27,7 +27,7 @@ require (
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
-	github.com/brianvoe/gofakeit/v7 v7.0.4
+	github.com/brianvoe/gofakeit/v7 v7.1.1
 	github.com/charmbracelet/bubbles v0.18.0
 	github.com/charmbracelet/bubbletea v0.27.0
 	github.com/charmbracelet/glamour v0.7.0
@@ -38,10 +38,10 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/go-elasticsearch/v8 v8.15.0
 	github.com/envoyproxy/protoc-gen-validate v1.1.0
-	github.com/fatih/color v1.17.0
+	github.com/fatih/color v1.18.0
 	github.com/felixge/fgprof v0.9.5
-	github.com/gabriel-vasile/mimetype v1.4.5
-	github.com/getsentry/sentry-go v0.29.0
+	github.com/gabriel-vasile/mimetype v1.4.6
+	github.com/getsentry/sentry-go v0.29.1
 	github.com/go-errors/errors v1.5.1
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/go-ldap/ldap/v3 v3.4.8
@@ -54,13 +54,13 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.20.2
-	github.com/google/go-github/v63 v63.0.0
+	github.com/google/go-github/v66 v66.0.0
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.13.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
-	github.com/jedib0t/go-pretty/v6 v6.6.0
+	github.com/jedib0t/go-pretty/v6 v6.6.1
 	github.com/jlaffaye/ftp v0.2.0
 	github.com/joho/godotenv v1.5.1
 	github.com/jpillora/overseer v1.1.6
@@ -78,38 +78,38 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.20.4
+	github.com/prometheus/client_golang v1.20.5
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/sassoftware/go-rpmutils v0.4.0
-	github.com/schollz/progressbar/v3 v3.16.1
+	github.com/schollz/progressbar/v3 v3.17.0
 	github.com/sendgrid/sendgrid-go v3.16.0+incompatible
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3
 	github.com/shuheiktgw/go-travis v0.3.1
 	github.com/snowflakedb/gosnowflake v1.11.2
 	github.com/stretchr/testify v1.9.0
-	github.com/tailscale/depaware v0.0.0-20240804103531-585336c3e1b3
-	github.com/testcontainers/testcontainers-go v0.33.0
-	github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.33.0
-	github.com/testcontainers/testcontainers-go/modules/mongodb v0.33.0
-	github.com/testcontainers/testcontainers-go/modules/mssql v0.33.0
-	github.com/testcontainers/testcontainers-go/modules/mysql v0.33.0
-	github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0
+	github.com/tailscale/depaware v0.0.0-20241028160002-3d7f3b30ed0e
+	github.com/testcontainers/testcontainers-go v0.34.0
+	github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0
+	github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0
+	github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0
+	github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0
+	github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0
 	github.com/trufflesecurity/disk-buffer-reader v0.2.1
 	github.com/wasilibs/go-re2 v1.7.0
-	github.com/xanzy/go-gitlab v0.110.0
+	github.com/xanzy/go-gitlab v0.112.0
 	github.com/xo/dburl v0.23.2
 	go.mongodb.org/mongo-driver v1.17.1
 	go.uber.org/automaxprocs v1.6.0
-	go.uber.org/mock v0.4.0
+	go.uber.org/mock v0.5.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.28.0
-	golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6
+	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
 	golang.org/x/net v0.30.0
 	golang.org/x/oauth2 v0.23.0
 	golang.org/x/sync v0.8.0
 	golang.org/x/text v0.19.0
-	google.golang.org/api v0.199.0
-	google.golang.org/protobuf v1.34.2
+	google.golang.org/api v0.203.0
+	google.golang.org/protobuf v1.35.1
 	gopkg.in/h2non/gock.v1 v1.1.2
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -120,13 +120,13 @@ require (
 
 require (
 	cel.dev/expr v0.16.1 // indirect
-	cloud.google.com/go v0.115.1 // indirect
-	cloud.google.com/go/auth v0.9.5 // indirect
+	cloud.google.com/go v0.116.0 // indirect
+	cloud.google.com/go/auth v0.9.9 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	cloud.google.com/go/iam v1.2.1 // indirect
 	cloud.google.com/go/longrunning v0.6.1 // indirect
-	cloud.google.com/go/monitoring v1.21.0 // indirect
+	cloud.google.com/go/monitoring v1.21.1 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
@@ -178,7 +178,7 @@ require (
 	github.com/couchbase/gocbcoreps v0.1.3 // indirect
 	github.com/couchbase/goprotostellar v1.0.2 // indirect
 	github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 // indirect
-	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -221,7 +221,7 @@ require (
 	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
@@ -242,7 +242,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.25 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -316,13 +316,13 @@ require (
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.25.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
-	google.golang.org/grpc v1.67.0 // indirect
+	google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
+	google.golang.org/grpc v1.67.1 // indirect
 	google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	pault.ag/go/topsort v0.1.1 // indirect