Improve Gitleaks Generic API Rule (#3009)

* Improve gitleaks generic api rule * Improve gitleaks generic api rule * fix indent * fix test case * fix test case * fix test todo * add original rule
semgrep · Aug 1, 2023 · a3d70f8 · a3d70f8
1 parent 5ce59a9
commit a3d70f8
Show file tree

Hide file tree

Showing 2 changed files with 85 additions and 12 deletions.
diff --git a/generic/secrets/gitleaks/generic-api-key.txt b/generic/secrets/gitleaks/generic-api-key.txt
@@ -6,22 +6,69 @@ generic_api_token = "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"
 "client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"
 // ruleid: generic-api-key
 "client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde"
-
+{"user": {
+  // ruleid: generic-api-key
+  "client_secret": CLOJARS_34bf0e88955ff5a1c328d6a7491acc4f48e865a7b8dd4d70a70749037443
+}}
 // ruleid: generic-api-key
 private const string UserCreationPasswordSecretKey = "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde";
 // ruleid: generic-api-key
-private const string UserCreationPasswordSecretKey = @"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde";
+private const string UserCreationPasswordSecretKey =@"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde";
+// ruleid: generic-api-key
+app.secret=edf10572-880c-4dd9-aaf0-6ec402f678db
+// ruleid: generic-api-key
+val PASSWORD = "Iv1.6213212547e00438__globPaths__123"
+eironment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: as2user
+      // ruleid: generic-api-key
+      POSTGRES_PASSWORD: eEEkp7Bb7q3xgL
+// ruleid: generic-api-key
+const DEFAULT_CLIENT_ID = 'aebc6443-996d-45c2-90f0-388ff96faa56';
+'roles' => 'ROLE_SUPER_ADMIN'
+val PASSWORD = "__globPaths__"
+
+ "lastModifiedSecret": 1556312220.133
+this.cmfPassword.foo = "thiscmfPassword1"
+
+const connectionToken = `12345-123-abc`;
+ this._perfKey = 'network_XMLHttpRequest_' + String(friendlyName);
+
+// todoruleid: generic-api-key
+this.txtCfmPassword.Name = "txtCfmPassword";
+
 // ok: generic-api-key
 private const string UserCreationPasswordSecretKey = @"Password";
+// ok: generic-api-key
+cache-key: flutter-3.3.x
+// ok: generic-api-key
+var key = _step2.value.key;
+// ok: generic-api-key
+"nextToken": "4AEA6u7J...The full token has been omitted for brevity...MzY2OA==",
+ttpXhrBackend.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "15.0.0", ngImport: i0,     
+
+
+ def zookeeperClient: KafkaZkClient = {
 
+  type: HttpXhrBackend, deps: [{ token: i1.XhrFactory }], target: i0.ɵɵFactoryTarget.Injectable }); 
 
 // ok: generic-api-key
+'Accept': 'application/json;api-version=3.0-preview.1',
+// ok: generic-api-key
+if (keyCode === wysihtml5.ENTER_KEY && !wysihtml5.browser.insertsLineBreaksOnReturn()) {
+
+// ok: generic-api-key
+# => #<Dalli::Client:0x007f98a47d2028 @servers=["127.0.0.1:11211"], @options={},         
+  @ring=nil>                
+// ok: generic-api-key
 newPassword=this.mPassword
 // ok: generic-api-key
 client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id
 // ok: generic-api-key
 password combination. R5: Regulatory--21
 
+// ok: generic-api-key
+password: 'K1f...........'
 / ok: generic-api-key
 newPassword=this.mPassword
 // ok: generic-api-key
@@ -42,6 +89,15 @@ SLACK_BOT_TOKEN=xoxb-0000000000-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
   "port": 8081
 }
 
+// ok:  generic-api-key
+author.author_address_id = 9223372036854775808 # out of range in the bigint
+
+"lastModifiedSecret": 1556312220.133
+
+"ObjectKey": "ami-1234567890abcdef0.bin"
+
+"ClientIP": "198.51.100.08"
+
 // todook: generic-api-key
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA=
 
@@ -72,7 +128,6 @@ IMAGER_S3_KEY=AWS_S3_KEY
 // ok
 x.MaxKey = mongodb.MaxKey;
 
-
 // ok
 User.findOne({ 'token': req.query.token }).exec(function(err, user)
 
@@ -94,8 +149,13 @@ qs: {
     'api-version': '2017-11-11-Preview'
 },
 
+const Accept = isWeb ? 'api-version=6.1-preview.1' : '*/*;api-version=4.0-preview.1';
+
+if (key === TOGGLE_DEV_TOOLS_KB || key === TOGGLE_DEV_TOOLS_KB_ALT) {
+
 // ok: generic-api-key
 GOOGLE_SECRET=<SECRET>
+// ok: generic-api-key
 IMAGER_S3_KEY=AWS_S3_KEY
 
 

diff --git a/generic/secrets/gitleaks/generic-api-key.yaml b/generic/secrets/gitleaks/generic-api-key.yaml
@@ -1,6 +1,12 @@
 rules:
 - id: generic-api-key
-  message: A gitleaks generic-api-key was detected which attempts to identify hard-coded credentials. It is not recommended to store credentials in source-code, as this risks secrets being leaked and used by either an internal or external malicious adversary. It is recommended to use environment variables to securely provide credentials or retrieve credentials from a secure vault or HSM (Hardware Security Module). This rule can introduce a lot of false positives, it is not recommended to be used in PR comments.
+  message: >-
+    A gitleaks generic-api-key was detected which attempts to identify hard-coded credentials. 
+    It is not recommended to store credentials in source-code, as this risks secrets being leaked 
+    and used by either an internal or external malicious adversary. It is recommended to use 
+    environment variables to securely provide credentials or retrieve credentials from a 
+    secure vault or HSM (Hardware Security Module). This rule can introduce a lot of false positives, 
+    it is not recommended to be used in PR comments.
   languages:
     - regex
   severity: INFO
@@ -33,24 +39,31 @@ rules:
       - "*/openssl/*.h"
       - "*.xcscmblueprint"
   patterns:
+    # The original regex from gitleaks is in this rule https://semgrep.dev/playground/s/57qk (but its very noisy) even with our entropy analyzer
     # This will likely remove some true positives, but this rule is overly noisy
     # Added (?-s) to prevent multi-lines with . which was causing a lot of FPs
+    # The only thing which has changed from the actual regex of gitleaks is adding in (?!([a-z]+\.[a-zA-Z]+)|.*(\d{4}-\d{2}-\d{2}|[a-z]+-[a-z]+.*)|:*(?!("|'))[0-9A-Za-z]+\.[0-9A-Za-z]+,|[A-Z]+_[A-Z]+_)
+    # We also added a capture group around the 'content' so we can 
     # added negative lookaheads to remove:
       # [a-z]+\.[a-zA-Z]+ (this.valueValue)
       # .* 
         # \d{4}-\d{2}-\d{2} (2017/03/12)
         # [a-z]+-[a-z]+.*. abc123-abc123 
         # :*(?!("|'))[0-9A-Za-z]+\.[0-9A-Za-z]+,  : 0123.0312abc, 
         # [A-Z]+_[A-Z]+_ VALUE_VALUE_
-    - pattern-regex: (?i)(?-s)(?:key|api|token|secret|client|passwd|password|auth|access).(?:[0-9a-z\-_\t
-        .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:).(?:'|\"|@|\s|=|\x60){0,5}(?!([a-z]+\.[a-zA-Z]+)|.*(\d{4}-\d{2}-\d{2}|[a-z]+-[a-z]+.*)|:*(?!("|'))[0-9A-Za-z]+\.[0-9A-Za-z]+,|[A-Z]+_[A-Z]+_)(?P<CONTENT>[0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)
+    - pattern-regex: (?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|@\"|\"|\s|=|\x60){0,5}(?!([a-z]+\.[a-zA-Z]+)|.*(\d{4}-\d{2}-\d{2}|[a-z]+-[a-z]+.*)|:*(?!("|'))[0-9A-Za-z]+\.[0-9A-Za-z]+,|[A-Z]+_[A-Z]+_)(?P<CONTENT>[0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)
     - metavariable-analysis:
         analyzer: entropy
         metavariable: $CONTENT
     - focus-metavariable: $CONTENT
-      # These remove test examples in addition to public keys, author= etc. 
-    - pattern-not-regex: (?i)publickeytoken=.*
-    - pattern-not-regex: (?i)(?:"|')pub
-    - pattern-not-regex: pubkey.*
-    - pattern-not-regex: ((token-drop|asset_key)("|'):.*0x)
-    - pattern-not-regex: (?i)(keywords|xxxx|eeeeeeee|0000|\*\*\*|example|test|public.*key|\.json|author=|author("|'))
+      # These remove values from the 'entire line so it could be the PublicKey=Something' could cause false negatives
+    - pattern-not-regex: .*((?i)omitted|arn:aws|(?i)(pub.*key|public.*key)|(?i)clientToken|symbol|cache|author\.).*
+      # These remove keywords or ip addresses from the content so only inside "PASSWORDEXAMPLE" its generic so anywhere 'inside' the $CONTENT
+    - pattern-not-regex: (\d\.\d\.\d-}|([\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3})|(\w)\1{5}|(?i)keywords|xxxx|eeeeeeee|0000|\*\*\*|example|test|author=|author("|')|preview|[A-Z]+_KEY|[.]value|[.]key|-\d\.\d\.)
+      # These are start or end checks e.g. starts as a hex code, ends with .json or starts with abcd or 12345 which usually indicates example code.
+    - metavariable-regex:
+        metavariable: $CONTENT
+        regex: (?!(^0x0*|^pub)|.*\.(bin|json|exe)$|.*(?i)(Client|Factory)$|(^__[A-Za-z]+__$)|^(12345|abcd)|^\d+(\.\d+)?$)
+       # Remove AAAAA, BBBBB, CCCCC, and .....
+    - pattern-not-regex: (\w|\.)\1{5}
+