From af217c9878fa4db47f9b570bdfeb3a4b387ac9fe Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Tue, 19 Nov 2024 21:30:49 -0500 Subject: [PATCH] feat(azure_search): update logic --- .../azuresearchadminkey.go | 106 ++++++++++-------- .../azuresearchadminkey_test.go | 55 +++++---- 2 files changed, 97 insertions(+), 64 deletions(-) diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go index 5cb0cf5792fa..5cc99e1a1b9d 100644 --- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go +++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go @@ -3,8 +3,8 @@ package azuresearchadminkey import ( "context" "fmt" + "io" "net/http" - "strings" regexp "github.com/wasilibs/go-re2" @@ -23,39 +23,49 @@ var _ detectors.Detector = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{52})\b`) - servicePat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{7,40})\b`) + + servicePat = regexp.MustCompile(`\b([a-z0-9][a-z0-9-]{5,58}[a-z0-9])\.search\.windows\.net`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "windows.net"}) + `\b([a-zA-Z0-9]{52})\b`) ) // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"azure"} + return []string{"search.windows.net"} +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_AzureSearchAdminKey +} + +func (s Scanner) Description() string { + return "Azure Search is a search-as-a-service solution that allows developers to incorporate search capabilities into their applications. Azure Search Admin Keys can be used to manage and query search services." } // FromData will find and optionally verify AzureSearchAdminKey secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - serviceMatches := servicePat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { + // Deduplicate results. + serviceMatches := make(map[string]struct{}) + for _, matches := range servicePat.FindAllStringSubmatch(dataStr, -1) { + serviceMatches[matches[1]] = struct{}{} + } + keyMatches := make(map[string]struct{}) + for _, matches := range keyPat.FindAllStringSubmatch(dataStr, -1) { + k := matches[1] + if detectors.StringShannonEntropy(k) < 4 { continue } - resMatch := strings.TrimSpace(match[1]) - for _, serviceMatch := range serviceMatches { - if len(serviceMatch) != 2 { - continue - } - resServiceMatch := strings.TrimSpace(serviceMatch[1]) + keyMatches[k] = struct{}{} + } - s1 := detectors.Result{ + for key := range keyMatches { + for service := range serviceMatches { + r := detectors.Result{ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resServiceMatch), + Raw: []byte(key), + RawV2: []byte(`{"service":"` + service + `","key":"` + key + `"}`), } if verify { @@ -63,39 +73,45 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if client == nil { client = defaultClient } - req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resServiceMatch+".search.windows.net/servicestats?api-version=2023-10-01-Preview", nil) - if err != nil { - continue - } - req.Header.Add("api-key", resMatch) - - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else if res.StatusCode == 401 || res.StatusCode == 403 { - // The secret is determinately not verified (nothing to do) - } else { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resMatch) - } - } else { - s1.SetVerificationError(err, resMatch) - } + + isVerified, verificationErr := verifyMatch(ctx, client, service, key) + r.Verified = isVerified + r.SetVerificationError(verificationErr, key) } - results = append(results, s1) + results = append(results, r) + if r.Verified { + break + } } } return results, nil } -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_AzureSearchAdminKey -} +func verifyMatch(ctx context.Context, client *http.Client, service string, key string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+service+".search.windows.net/servicestats?api-version=2023-10-01-Preview", nil) + if err != nil { + return false, err + } + req.Header.Set("api-key", key) -func (s Scanner) Description() string { - return "Azure Search is a search-as-a-service solution that allows developers to incorporate search capabilities into their applications. Azure Search Admin Keys can be used to manage and query search services." + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + // The secret is determinately not verified. + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } } diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go index aeb5b2094b1f..0deb9d6f0e6b 100644 --- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go +++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go @@ -10,19 +10,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -var ( - validPattern = ` - azure: - azureKey: wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJyma - azureService: TestingService01 - ` - invalidPattern = ` - azure: - Key: wRRPyhjv8m6JGRujUUr-PK#a8d3rJ0mrGAxhmqf3A68OgZmlWUJyma - Service: TS01 - ` -) - func TestAzureSearchAdminKey_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) @@ -33,14 +20,44 @@ func TestAzureSearchAdminKey_Pattern(t *testing.T) { want []string }{ { - name: "valid pattern", - input: validPattern, - want: []string{"wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaTestingService01", "wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaazureKey"}, + name: "valid pattern", + input: ` + azure: + azureKey: wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJyma + azureService: testingservice01.search.windows.net + `, + want: []string{`{"service":"testingservice01","key":"wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJyma"}`}, + }, + { + name: "jupyter notebook", + input: ` + { + "cell_type": "code", + "execution_count": 7, + "id": "b188568f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://news-search.search.windows.net azureblob-index 2021-04-30-Preview iOExxhJ2A2wjdAGTjsMxASsJj3y3zUR54kjNTNpW9hAzSeD8PE3k\n" + ] + } + ], + "source": [ + "print(os.getenv(\"AZURE_COGNITIVE_SEARCH_ENDPOINT\"), os.getenv('AZURE_COGNITIVE_SEARCH_INDEX_NAME'), os.getenv('AZURE_COGNITIVE_SEARCH_API_VERSION'), os.getenv(\"AZURE_COGNITIVE_SEARCH_KEY\"))" + ]`, + want: []string{`{"service":"news-search","key":"iOExxhJ2A2wjdAGTjsMxASsJj3y3zUR54kjNTNpW9hAzSeD8PE3k"}`}, }, { - name: "invalid pattern", - input: invalidPattern, - want: nil, + name: "invalid pattern", + input: ` + azure: + Key: wRRPyhjv8m6JGRujUUr-PK#a8d3rJ0mrGAxhmqf3A68OgZmlWUJyma + Service: TS01.search.windows.net + `, + want: nil, }, }