更新 nuclei、sensitive keys;一些错误检查正则预编译

go.mod

@@ -65,7 +65,7 @@ require (
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/katana v1.0.6-0.20240313185050-24c31f49c050
 	github.com/projectdiscovery/naabu/v2 v2.3.0
-	github.com/projectdiscovery/nuclei/v3 v3.2.1
+	github.com/projectdiscovery/nuclei/v3 v3.2.2
 	github.com/projectdiscovery/retryabledns v1.0.58
 	github.com/samber/lo v1.39.0
 	github.com/sergi/go-diff v1.3.1

go.sum

@@ -1027,6 +1027,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
 github.com/projectdiscovery/nuclei/v3 v3.2.1 h1:p4Cg1i/rFysMeXnYBY28s6AWb1eZj6pXkWTNREFcaPA=
 github.com/projectdiscovery/nuclei/v3 v3.2.1/go.mod h1:LkKLQeiQRavbZAnpDCP1LWOC7854OmNwRn2Z+YwH/ME=
+github.com/projectdiscovery/nuclei/v3 v3.2.2 h1:bCa0pW2EKVHxdx+2kKMJOlGftPzH0JCSrt1a0oh6H60=
+github.com/projectdiscovery/nuclei/v3 v3.2.2/go.mod h1:LkKLQeiQRavbZAnpDCP1LWOC7854OmNwRn2Z+YwH/ME=
 github.com/projectdiscovery/ratelimit v0.0.33 h1:MT8Oa0VVBBI5w6ZMUJCIIQkjdTVNbzhGRDMrNqV1BQ4=
 github.com/projectdiscovery/ratelimit v0.0.33/go.mod h1:Mdbm5Olxd0zddUO3Khy330H1Ei7377/DFIuY9nRZuGM=
 github.com/projectdiscovery/rawhttp v0.1.41 h1:0n6CohOf0Aq7dsXv+ozznhlYr4ANDKLwvPmdzTet3qU=

scan/gadget/sensitive/error.go

@@ -65,6 +65,24 @@ var errors = []ErrorMessage{
 
 var seenRequests sync.Map // 这里主要是为了一些返回包检测类的判断是否识别过，减小开销，扫描类内部会判断是否扫描过
 
+type Regexp struct {
+    Re  *regexp.Regexp
+    Msg ErrorMessage
+}
+
+var errorCompiled map[string]*Regexp
+
+func init() {
+    // 只编译一次编译正则
+    errorCompiled = make(map[string]*Regexp, len(errors))
+    for _, errorMsg := range errors {
+        errorCompiled[errorMsg.Text] = &Regexp{
+            Re:  regexp.MustCompile(errorMsg.Text),
+            Msg: errorMsg,
+        }
+    }
+}
+
 func PageErrorMessageCheck(url, req, body string) []ErrorMessage {
     // 因为放到了 httpx.Request 中，所以会有很多重复，这里检验一下 url 是否已经检测过了
     if _, ok := seenRequests.Load(url); ok {
@@ -73,34 +91,34 @@ func PageErrorMessageCheck(url, req, body string) []ErrorMessage {
     seenRequests.Store(url, true)
 
     var results []ErrorMessage
-    for _, errorMsg := range errors {
-        re := regexp.MustCompile(errorMsg.Text)
+    for _, errorMsg := range errorCompiled {
+        re := errorMsg.Re
         result := re.FindString(body)
         if result != "" {
             // org.springframework.web.HttpRequestMethodNotSupportedException 这种也会匹配到，java 这样的会误报混淆
-            if "([A-Za-z]+[.])+[A-Za-z]*Exception: " == errorMsg.Text && strings.Contains(body, ".java") {
+            if "([A-Za-z]+[.])+[A-Za-z]*Exception: " == errorMsg.Msg.Text && strings.Contains(body, ".java") {
                 continue
             }
 
             results = append(results, ErrorMessage{
                 Text: result,
-                Type: errorMsg.Type,
+                Type: errorMsg.Msg.Type,
             })
 
             output.OutChannel <- output.VulMessage{
                 DataType: "web_vul",
                 Plugin:   "Sensitive error",
                 VulnData: output.VulnData{
                     CreateTime: time.Now().Format("2006-01-02 15:04:05"),
-                    VulnType:   errorMsg.Text,
+                    VulnType:   errorMsg.Msg.Text,
                     Target:     url,
                     Payload:    result,
                     Request:    req,
                     Response:   body,
                 },
                 Level: output.Low,
             }
-            logging.Logger.Infoln("[Sensitive]", url, errorMsg.Type, result)
+            logging.Logger.Infoln("[Sensitive]", url, errorMsg.Msg.Type, result)
         }
     }
 

scan/gadget/sensitive/key.go

@@ -17,7 +17,7 @@ import (
   @desc: 提取 https://github.com/projectdiscovery/nuclei-templates/tree/main/file/keys 中的规则
 **/
 
-//go:embed rules/*
+//go:embed keys/*
 var ruleFiles embed.FS
 var rules []templates.Template
 

scan/gadget/sensitive/keys/adafruit-key.yaml

@@ -0,0 +1,23 @@
+id: adafruit-key
+
+info:
+  name: Adafruit API Key
+  author: DhiyaneshDK
+  severity: info
+  reference:
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/adafruit-api-key.yaml
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/adafruit-api-key.go
+  metadata:
+    verified: true
+  tags: adafruit,file,keys
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - (?i)(?:adafruit)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)
+
+# digest: 4a0a00473045022100e18e66c25918d1d8e980ab39a1d206e65dc34ef8b6ae0e043c87d34f0496d4260220651cd87fb75b897e27766f354e0711534ef67b6f368885d00fbf79ed44ed72a7:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/adobe/adobe-client.yaml

@@ -0,0 +1,23 @@
+id: adobe-client
+
+info:
+  name: Adobe Client ID
+  author: DhiyaneshDK
+  severity: info
+  reference:
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/adobe-client-id.yaml
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/adobe-client-id.go
+  metadata:
+    verified: true
+  tags: keys,file,adobe,token
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - (?i)(?:adobe)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)
+# digest: 490a00463044022007eda94aded10055c992548f92f163ce142cfa63312df87ab1913d55655c84a402205cfb63b7803c40be56e370f98a2541ef20c37455b0b0f136a5c19164ee802429:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/rules/adobe-secret.yaml → scan/gadget/sensitive/keys/adobe/adobe-secret.yaml

@@ -10,8 +10,8 @@ info:
     - https://developer.adobe.com/developer-console/docs/guides/authentication/OAuthIntegration/
     - https://developer.adobe.com/developer-console/docs/guides/authentication/OAuth/
   metadata:
-    verified: "true"
-  tags: adobe,oauth,file,token
+    verified: true
+  tags: file,keys,adobe,oauth,token
 
 file:
   - extensions:
@@ -21,4 +21,5 @@ file:
       - type: regex
         part: body
         regex:
-          - '(?i)\b(p8e-[a-z0-9-]{32})(?:[^a-z0-9-]|$)'
+          - '(?i)\b(p8e-[a-z0-9-]{32})(?:[^a-z0-9-]|$)'
+# digest: 4a0a00473045022100fbb2a00c904fe46b3138bc5a79cd5d3e108bf9a7ce64db4d82a47a40b4edfc7e022036f0b1d84e6bbde773bd90b9021e8202465c54346d9f1436af84e622a119114a:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/rules/age/age-identity-secret-key.yaml → scan/gadget/sensitive/keys/age/age-identity-secret-key.yaml

@@ -9,8 +9,8 @@ info:
     - https://github.com/FiloSottile/age/blob/main/doc/age.1.html
     - https://github.com/C2SP/C2SP/blob/8b6a842e0360d35111c46be2a8019b2276295914/age.md#the-x25519-recipient-type
   metadata:
-    verified: "true"
-  tags: age-encryption,file,token
+    verified: true
+  tags: file,keys,age-encryption,token
 
 file:
   - extensions:
@@ -20,4 +20,5 @@ file:
       - type: regex
         part: body
         regex:
-          - '\bAGE-SECRET-KEY-1[0-9A-Z]{58}\b'
+          - '\bAGE-SECRET-KEY-1[0-9A-Z]{58}\b'
+# digest: 4a0a00473045022100967a33608a1ecaa232719a64590ae179e82473d9ff9960e1294033f41dcfafb3022011659ec4586dff37d9381700897e858d37c2b363d718315d96fa9db721bc7123:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/rules/age/age-recipient-public-key.yaml → scan/gadget/sensitive/keys/age/age-recipient-public-key.yaml

@@ -9,8 +9,8 @@ info:
     - https://github.com/FiloSottile/age/blob/main/doc/age.1.html
     - https://github.com/C2SP/C2SP/blob/8b6a842e0360d35111c46be2a8019b2276295914/age.md#the-x25519-recipient-type
   metadata:
-    verified: "true"
-  tags: age-encryption,file,token
+    verified: true
+  tags: file,keys,age-encryption,token
 
 file:
   - extensions:
@@ -20,4 +20,5 @@ file:
       - type: regex
         part: body
         regex:
-          - '\bage1[0-9a-z]{58}\b'
+          - '\bage1[0-9a-z]{58}\b'
+# digest: 4b0a004830460221008efb372243352ac7767832750aa04221c747bfb407e0d3599f6716055832807402210084c3968cf28f080a9a1ef95e6cd8a9029e85c7fa0d051df56217ecc16d6aafb9:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/airtable-key.yaml

@@ -0,0 +1,22 @@
+id: airtable-key
+
+info:
+  name: Airtable API Key
+  author: DhiyaneshDK
+  severity: info
+  reference:
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/airtable-api-key.yaml
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/airtable-api-key.go
+  metadata:
+    verified: true
+  tags: keys,file,airtable,token
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - (?i)(?:airtable)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{17})(?:['|\"|\n|\r|\s|\x60|;]|$)
+# digest: 490a004630440220673067de4dbbe1d9d4f9337d2eddd6903ed401646b5e2ef23b4cb4fbc15e4bb40220774a7aafc56f3023bd7d681d429badb45d714352a8fcb74844e5913b116cfce2:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/algolia-key.yaml

@@ -0,0 +1,23 @@
+id: algolia-key
+
+info:
+  name: Algolia API Key
+  author: DhiyaneshDK
+  severity: info
+  reference:
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/algolia-api-key.yaml
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/algolia-api-key.go
+  metadata:
+    verified: true
+  tags: algolia,file,keys
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - (?i)(?:algolia)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)
+
+# digest: 4a0a0047304502200114ce7db1c3fde42b20020e1d0ccddb88507568c665f21e1cdc8a7b722defdb022100c707d824ef36106683f16cc962e32ac899c727c5b22db59a7af8a4ab957a27d6:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/alibaba/alibaba-key-id.yaml

@@ -0,0 +1,23 @@
+id: alibaba-key-id
+
+info:
+  name: Alibaba Access Key ID
+  author: DhiyaneshDK
+  severity: info
+  reference:
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/alibaba-access-key-id.yaml
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/alibaba-access-key-id.go
+  metadata:
+    verified: true
+  tags: alibaba,access,file,keys
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - (?i)\b((LTAI)(?i)[a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)
+# digest: 490a0046304402202a929c5a7c56fdcba6baf8a05f5ee26de1dc68039a330a33dba7e6973876605b0220499fe8d24c2d03e30f7ffa4077775380ea6b237262bfdc1319821135d3bf0faf:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/alibaba/alibaba-secret-id.yaml

@@ -0,0 +1,23 @@
+id: alibaba-secret-id
+
+info:
+  name: Alibaba Secret Key ID
+  author: DhiyaneshDK
+  severity: info
+  reference:
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/alibaba-secret-key.yaml
+    - https://github.com/returntocorp/semgrep-rules/blob/develop/generic/secrets/gitleaks/alibaba-secret-key.go
+  metadata:
+    verified: true
+  tags: alibaba,secret,file,keys
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - (?i)(?:alibaba)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)
+# digest: 4b0a0048304602210087f98e454e5064757753028db3f4a280d96ee2ba47163b503031bb9000820d73022100f8348ca58ad2ee80dba4b7ccbca37a95b7ba44742a4f0ed2f5fd64b952843ef1:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/amazon/amazon-account-id.yaml

@@ -0,0 +1,29 @@
+id: amazon-account-id
+
+info:
+  name: Amazon Web Services Account ID - Detect
+  author: DhiyaneshDK
+  severity: info
+  description: Amazon Web Services Account ID token was detected.
+  reference:
+    - https://github.com/praetorian-inc/noseyparker/blob/main/data/default/rules/aws.yml
+  classification:
+    cvss-metrics: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N
+    cvss-score: 0
+    cwe-id: CWE-200
+  metadata:
+    verified: true
+  tags: file,keys,aws,amazon,token
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - '(?i)aws_?(?:account)_?(?:id)?["''`]?\s{0,30}(?::|=>|=)\s{0,30}["''`]?([0-9]{4}-?[0-9]{4}-?[0-9]{4})'
+
+# Enhanced by md on 2023/05/04
+# digest: 4b0a00483046022100ad930551f3063ad8ee7027d7e0af408452b42a4dc33ba7a99e5bcbcf845c7e05022100b1d4fcc47c2ae007d17b06c945a91c56d8f4f5166d69688d8707bc4fcb69266e:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/amazon/amazon-mws-auth-token.yaml

@@ -0,0 +1,22 @@
+id: amazon-mws-auth-token-value
+
+info:
+  name: Amazon MWS Authentication Token - Detect
+  author: gaurang
+  severity: medium
+  description: Amazon MWS authentication token was detected.
+  classification:
+    cvss-metrics: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N
+    cvss-score: 5.3
+    cwe-id: CWE-200
+  tags: file,keys,token,amazon,auth,mws
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        regex:
+          - "amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+# digest: 4a0a00473045022100c334a6bda970ddcb70079df2f8a9a1769a7104636a611691c28787921fc2a1a102200bfe666c925c702093688b5f70b29028fa8c8c92c8b739cee1eaaa3a92144494:922c64590222798bb761d5b6d8e72950

scan/gadget/sensitive/keys/amazon/amazon-session-token.yaml

@@ -0,0 +1,29 @@
+id: amazon-session-token
+
+info:
+  name: Amazon Session Token - Detect
+  author: DhiyaneshDK
+  severity: info
+  description: Amazon session token was detected.
+  reference:
+    - https://github.com/praetorian-inc/noseyparker/blob/main/data/default/rules/aws.yml
+  classification:
+    cvss-metrics: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N
+    cvss-score: 0
+    cwe-id: CWE-200
+  metadata:
+    verified: true
+  tags: file,keys,aws,amazon,token,session
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - '(?i)(?:aws.?session|aws.?session.?token|aws.?token)["''`]?\s{0,30}(?::|=>|=)\s{0,30}["''`]?([a-z0-9/+=]{16,200})[^a-z0-9/+=]'
+
+# Enhanced by md on 2023/05/04
+# digest: 4a0a00473045022012a50d46848dcc172a05c5e2fd88e802af8022bf13ab09dbf8740ae3ad5855f5022100c16953404125451a8cfc4ed26412b99b0d25c02e73a6c7ba8337a905c7e2efa9:922c64590222798bb761d5b6d8e72950