Enhance secret scanning API calls to return more data (#87)

* Enhance secret scanning API calls to return more data

* run black

* black with superlinter

* Add Black configuration for code formatting to pyproject.toml so it can be used local with black (and picked up by superlinter)

* Fix super-linter configuration to ensure Black uses the correct pyproject.toml file

* Add Black configuration to super-linter and ensure proper validation

Author: felickz

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.black]
+line-length = 120

src/secret_scanning.py

@@ -18,12 +18,12 @@ def get_repo_ss_alerts(api_endpoint, github_pat, repo_name):
     - List of _all_ secret scanning alerts on the repository (both default and generic secret types)
     """
     # First call: get default secret types (without any filters), use after= to force object based cursor instead of page based
-    url_default = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after="
+    url_default = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=&hide_secret=true"
     ss_alerts_default = api_helpers.make_api_call(url_default, github_pat)
 
     # Second call: get generic secret types with hardcoded list, use after= to force object based cursor instead of page based
     generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key"
-    url_generic = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}"
+    url_generic = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}&hide_secret=true"
     ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat)
 
     # Combine results and deduplicate
@@ -73,31 +73,79 @@ def write_repo_ss_list(secrets_list):
             [
                 "number",
                 "created_at",
+                "updated_at",
                 "html_url",
                 "state",
                 "resolution",
                 "resolved_at",
                 "resolved_by_username",
                 "resolved_by_type",
                 "resolved_by_isadmin",
+                "resolution_comment",
                 "secret_type",
                 "secret_type_display_name",
+                "validity",
+                "publicly_leaked",
+                "multi_repo",
+                "is_base64_encoded",
+                "first_location_path",
+                "first_location_start_line",
+                "first_location_commit_sha",
+                "push_protection_bypassed",
+                "push_protection_bypassed_by",
+                "push_protection_bypassed_at",
+                "push_protection_bypass_request_reviewer",
+                "push_protection_bypass_request_reviewer_comment",
+                "push_protection_bypass_request_comment",
+                "push_protection_bypass_request_html_url",
+                "assigned_to",
             ]
         )
         for alert in secrets_list:
+            first_location = alert.get("first_location_detected") or {}
+
             writer.writerow(
                 [
                     alert["number"],
                     alert["created_at"],
+                    alert["updated_at"],
                     alert["html_url"],
                     alert["state"],
                     alert["resolution"],
                     alert["resolved_at"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["login"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["type"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"],
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["login"]),
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["type"]),
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"]),
+                    alert.get("resolution_comment", ""),
                     alert["secret_type"],
                     alert["secret_type_display_name"],
+                    alert["validity"],
+                    str(alert["publicly_leaked"]),
+                    str(alert["multi_repo"]),
+                    str(alert["is_base64_encoded"]),
+                    first_location.get("path")
+                    or first_location.get("pull_request_body_url")
+                    or first_location.get("issue_body_url")
+                    or first_location.get("discussion_body_url")
+                    or "",
+                    ("" if first_location is None else first_location.get("start_line", "")),
+                    ("" if first_location is None else first_location.get("commit_sha", "")),
+                    str(alert["push_protection_bypassed"]),
+                    (
+                        ""
+                        if alert.get("push_protection_bypassed_by") is None
+                        else alert["push_protection_bypassed_by"].get("login", "")
+                    ),
+                    alert.get("push_protection_bypassed_at", ""),
+                    (
+                        ""
+                        if alert.get("push_protection_bypass_request_reviewer") is None
+                        else alert["push_protection_bypass_request_reviewer"].get("login", "")
+                    ),
+                    alert.get("push_protection_bypass_request_reviewer_comment", ""),
+                    alert.get("push_protection_bypass_request_comment", ""),
+                    alert.get("push_protection_bypass_request_html_url", ""),
+                    ("" if alert.get("assigned_to") is None else alert["assigned_to"].get("login", "")),
                 ]
             )
 
@@ -115,32 +163,32 @@ def get_org_ss_alerts(api_endpoint, github_pat, org_name):
     - List of _all_ secret scanning alerts on the organization (both default and generic secret types)
     """
     # First call: get default secret types (without any filters), use after= to force object based cursor instead of page based
-    url_default = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after="
+    url_default = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=&hide_secret=true"
     ss_alerts_default = api_helpers.make_api_call(url_default, github_pat)
 
     # Second call: get generic secret types with hardcoded list, use after= to force object based cursor instead of page based
     generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key"
-    url_generic = (
-        f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}"
-    )
+    url_generic = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}&hide_secret=true"
     ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat)
 
-    # Combine results and deduplicate
+    # Combine results and deduplicate using composite key (repo + alert number)
     combined_alerts = []
-    alert_numbers_seen = set()
+    alert_keys_seen = set()  # Composite key: (repo, alert_number)
     duplicates_found = False
 
     # Add default alerts
     for alert in ss_alerts_default:
-        alert_numbers_seen.add(alert["number"])
+        alert_key = (alert["repository"]["full_name"], alert["number"])
+        alert_keys_seen.add(alert_key)
         combined_alerts.append(alert)
 
     # Add generic alerts, checking for duplicates
     for alert in ss_alerts_generic:
-        if alert["number"] in alert_numbers_seen:
+        alert_key = (alert["repository"]["full_name"], alert["number"])
+        if alert_key in alert_keys_seen:
             duplicates_found = True
         else:
-            alert_numbers_seen.add(alert["number"])
+            alert_keys_seen.add(alert_key)
             combined_alerts.append(alert)
 
     # Warn if duplicates were found
@@ -172,15 +220,32 @@ def write_org_ss_list(secrets_list):
             [
                 "number",
                 "created_at",
+                "updated_at",
                 "html_url",
                 "state",
                 "resolution",
                 "resolved_at",
                 "resolved_by_username",
                 "resolved_by_type",
                 "resolved_by_isadmin",
+                "resolution_comment",
                 "secret_type",
                 "secret_type_display_name",
+                "validity",
+                "publicly_leaked",
+                "multi_repo",
+                "is_base64_encoded",
+                "first_location_path",
+                "first_location_start_line",
+                "first_location_commit_sha",
+                "push_protection_bypassed",
+                "push_protection_bypassed_by",
+                "push_protection_bypassed_at",
+                "push_protection_bypass_request_reviewer",
+                "push_protection_bypass_request_reviewer_comment",
+                "push_protection_bypass_request_comment",
+                "push_protection_bypass_request_html_url",
+                "assigned_to",
                 "repo_name",
                 "repo_owner",
                 "repo_owner_type",
@@ -191,19 +256,50 @@ def write_org_ss_list(secrets_list):
             ]
         )
         for alert in secrets_list:
+            first_location = alert.get("first_location_detected") or {}
+
             writer.writerow(
                 [
                     alert["number"],
                     alert["created_at"],
+                    alert["updated_at"],
                     alert["html_url"],
                     alert["state"],
                     alert["resolution"],
                     alert["resolved_at"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["login"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["type"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"],
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["login"]),
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["type"]),
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"]),
+                    alert.get("resolution_comment", ""),
                     alert["secret_type"],
                     alert["secret_type_display_name"],
+                    alert["validity"],
+                    str(alert["publicly_leaked"]),
+                    str(alert["multi_repo"]),
+                    str(alert["is_base64_encoded"]),
+                    first_location.get("path")
+                    or first_location.get("pull_request_body_url")
+                    or first_location.get("issue_body_url")
+                    or first_location.get("discussion_body_url")
+                    or "",
+                    ("" if first_location is None else first_location.get("start_line", "")),
+                    ("" if first_location is None else first_location.get("commit_sha", "")),
+                    str(alert["push_protection_bypassed"]),
+                    (
+                        ""
+                        if alert.get("push_protection_bypassed_by") is None
+                        else alert["push_protection_bypassed_by"].get("login", "")
+                    ),
+                    alert.get("push_protection_bypassed_at", ""),
+                    (
+                        ""
+                        if alert.get("push_protection_bypass_request_reviewer") is None
+                        else alert["push_protection_bypass_request_reviewer"].get("login", "")
+                    ),
+                    alert.get("push_protection_bypass_request_reviewer_comment", ""),
+                    alert.get("push_protection_bypass_request_comment", ""),
+                    alert.get("push_protection_bypass_request_html_url", ""),
+                    ("" if alert.get("assigned_to") is None else alert["assigned_to"].get("login", "")),
                     alert["repository"]["full_name"],
                     alert["repository"]["owner"]["login"],
                     alert["repository"]["owner"]["type"],
@@ -229,30 +325,42 @@ def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug):
     - List of _all_ secret scanning alerts on the enterprise (both default and generic secret types)
     """
     # First call: get default secret types (without any filters), use after= to force object based cursor instead of page based
-    url_default = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after="
+    url_default = (
+        f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=&hide_secret=true"
+    )
     ss_alerts_default = api_helpers.make_api_call(url_default, github_pat)
 
     # Second call: get generic secret types with hardcoded list, use after= to force object based cursor instead of page based
     generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key"
-    url_generic = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}"
+    url_generic = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}&hide_secret=true"
     ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat)
 
-    # Combine results and deduplicate
+    # Combine results and deduplicate using composite key (org + repo + alert number)
     combined_alerts = []
-    alert_numbers_seen = set()
+    alert_keys_seen = set()  # Composite key: (org, repo, alert_number)
     duplicates_found = False
 
     # Add default alerts
     for alert in ss_alerts_default:
-        alert_numbers_seen.add(alert["number"])
+        alert_key = (
+            alert["repository"]["owner"]["login"],
+            alert["repository"]["name"],
+            alert["number"],
+        )
+        alert_keys_seen.add(alert_key)
         combined_alerts.append(alert)
 
     # Add generic alerts, checking for duplicates
     for alert in ss_alerts_generic:
-        if alert["number"] in alert_numbers_seen:
+        alert_key = (
+            alert["repository"]["owner"]["login"],
+            alert["repository"]["name"],
+            alert["number"],
+        )
+        if alert_key in alert_keys_seen:
             duplicates_found = True
         else:
-            alert_numbers_seen.add(alert["number"])
+            alert_keys_seen.add(alert_key)
             combined_alerts.append(alert)
 
     # Warn if duplicates were found
@@ -284,15 +392,32 @@ def write_enterprise_ss_list(secrets_list):
             [
                 "number",
                 "created_at",
+                "updated_at",
                 "html_url",
                 "state",
                 "resolution",
                 "resolved_at",
                 "resolved_by_username",
                 "resolved_by_type",
                 "resolved_by_isadmin",
+                "resolution_comment",
                 "secret_type",
                 "secret_type_display_name",
+                "validity",
+                "publicly_leaked",
+                "multi_repo",
+                "is_base64_encoded",
+                "first_location_path",
+                "first_location_start_line",
+                "first_location_commit_sha",
+                "push_protection_bypassed",
+                "push_protection_bypassed_by",
+                "push_protection_bypassed_at",
+                "push_protection_bypass_request_reviewer",
+                "push_protection_bypass_request_reviewer_comment",
+                "push_protection_bypass_request_comment",
+                "push_protection_bypass_request_html_url",
+                "assigned_to",
                 "repo_name",
                 "repo_owner",
                 "repo_owner_type",
@@ -303,19 +428,50 @@ def write_enterprise_ss_list(secrets_list):
             ]
         )
         for alert in secrets_list:
+            first_location = alert.get("first_location_detected") or {}
+
             writer.writerow(
                 [
                     alert["number"],
                     alert["created_at"],
+                    alert["updated_at"],
                     alert["html_url"],
                     alert["state"],
                     alert["resolution"],
                     alert["resolved_at"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["login"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["type"],
-                    "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"],
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["login"]),
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["type"]),
+                    ("" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"]),
+                    alert.get("resolution_comment", ""),
                     alert["secret_type"],
                     alert["secret_type_display_name"],
+                    alert["validity"],
+                    str(alert["publicly_leaked"]),
+                    str(alert["multi_repo"]),
+                    str(alert["is_base64_encoded"]),
+                    first_location.get("path")
+                    or first_location.get("pull_request_body_url")
+                    or first_location.get("issue_body_url")
+                    or first_location.get("discussion_body_url")
+                    or "",
+                    ("" if first_location is None else first_location.get("start_line", "")),
+                    ("" if first_location is None else first_location.get("commit_sha", "")),
+                    str(alert["push_protection_bypassed"]),
+                    (
+                        ""
+                        if alert.get("push_protection_bypassed_by") is None
+                        else alert["push_protection_bypassed_by"].get("login", "")
+                    ),
+                    alert.get("push_protection_bypassed_at", ""),
+                    (
+                        ""
+                        if alert.get("push_protection_bypass_request_reviewer") is None
+                        else alert["push_protection_bypass_request_reviewer"].get("login", "")
+                    ),
+                    alert.get("push_protection_bypass_request_reviewer_comment", ""),
+                    alert.get("push_protection_bypass_request_comment", ""),
+                    alert.get("push_protection_bypass_request_html_url", ""),
+                    ("" if alert.get("assigned_to") is None else alert["assigned_to"].get("login", "")),
                     alert["repository"]["full_name"],
                     alert["repository"]["owner"]["login"],
                     alert["repository"]["owner"]["type"],