forked from trufflesecurity/trufflehog
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement verification cache (trufflesecurity#3801)
This PR introduces a cache that allows the scanner to avoid emitting multiple requests to verify the same credential. In practice, it doesn't seem to reduce scan time at all, but it does seem to reduce the number of calls to FromData rather drastically. The cache is implemented as an opt-out feature that can be disabled with a new CLI flag. If we don't like this, we can change it. The metrics collection hopefully isn't too architecture-astronauty; I wanted to create something useful here that could also accommodate future Prometheus configuration without making the implementation all stupid.
- Loading branch information
Showing
10 changed files
with
546 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package verificationcache | ||
|
|
||
| import ( | ||
| "sync/atomic" | ||
| "time" | ||
| ) | ||
|
|
||
| // InMemoryMetrics is a MetricsReporter that stores reported metrics in memory for retrieval at the end of a scan. | ||
| type InMemoryMetrics struct { | ||
| CredentialVerificationsSaved atomic.Int32 | ||
| FromDataVerifyTimeSpentMS atomic.Int64 | ||
| ResultCacheHits atomic.Int32 | ||
| ResultCacheHitsWasted atomic.Int32 | ||
| ResultCacheMisses atomic.Int32 | ||
| } | ||
|
|
||
| var _ MetricsReporter = (*InMemoryMetrics)(nil) | ||
|
|
||
| func (m *InMemoryMetrics) AddCredentialVerificationsSaved(count int) { | ||
| m.CredentialVerificationsSaved.Add(int32(count)) | ||
| } | ||
|
|
||
| func (m *InMemoryMetrics) AddFromDataVerifyTimeSpent(wallTime time.Duration) { | ||
| m.FromDataVerifyTimeSpentMS.Add(wallTime.Milliseconds()) | ||
| } | ||
|
|
||
| func (m *InMemoryMetrics) AddResultCacheHits(count int) { | ||
| m.ResultCacheHits.Add(int32(count)) | ||
| } | ||
|
|
||
| func (m *InMemoryMetrics) AddResultCacheMisses(count int) { | ||
| m.ResultCacheMisses.Add(int32(count)) | ||
| } | ||
|
|
||
| func (m *InMemoryMetrics) AddResultCacheHitsWasted(count int) { | ||
| m.ResultCacheHitsWasted.Add(int32(count)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package verificationcache | ||
|
|
||
| import "time" | ||
|
|
||
| // MetricsReporter is an interface used by a verification cache to report various metrics related to its operation. | ||
| // Implementations must be thread-safe. | ||
| type MetricsReporter interface { | ||
| // AddCredentialVerificationsSaved records "saved" verification attempts, which is when credential verification | ||
| // status is loaded from the cache instead of retrieved from a remote verification endpoint. This number might be | ||
| // smaller than the cache hit count due to cache hit "wasting"; see AddResultCacheHitsWasted for more information. | ||
| AddCredentialVerificationsSaved(count int) | ||
|
|
||
| // AddFromDataVerifyTimeSpent records wall time spent in calls to detector.FromData with verify=true. | ||
| AddFromDataVerifyTimeSpent(wallTime time.Duration) | ||
|
|
||
| // AddResultCacheHits records result cache hits. Not all cache hits result in elided remote verification requests | ||
| // due to cache hit "wasting"; see AddResultCacheHitsWasted for more information. | ||
| AddResultCacheHits(count int) | ||
|
|
||
| // AddResultCacheMisses records result cache misses. | ||
| AddResultCacheMisses(count int) | ||
|
|
||
| // AddResultCacheHitsWasted records "wasted" result cache hits. A "wasted" result cache hit is a result cache hit | ||
| // that does not elide a remote verification request because there are other secret findings in the relevant chunk | ||
| // that are not cached. When this happens, the detector's FromData method must be called anyway, so the cache hit | ||
| // doesn't save any remote requests. | ||
| AddResultCacheHitsWasted(count int) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| package verificationcache | ||
|
|
||
| import ( | ||
| "github.com/trufflesecurity/trufflehog/v3/pkg/cache" | ||
| "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
| ) | ||
|
|
||
| // ResultCache is a cache that holds individual detector results. It serves as a component of a VerificationCache. | ||
| type ResultCache cache.Cache[detectors.Result] |
Oops, something went wrong.