Redesign secret scanning to use two API calls by default

Co-authored-by: felickz <1760475+felickz@users.noreply.github.com>

Author: Copilot

README.md

@@ -77,7 +77,6 @@ The list of all available options that can be set as environmental variables is
 - `GITHUB_REPORT_SCOPE`: The scope of the report to generate. Valid values are `repository` (default), `organization`  or `enterprise`.
 - `SCOPE_NAME` or `GITHUB_REPOSITORY`: The name of the repository, organization or enterprise to generate the report for. If `SCOPE_NAME` is not set, the value of `GITHUB_REPOSITORY` is used if it is set. If neither is set, an error occurs.
 - `FEATURES`: A comma-separated list of features to include in the report. Valid values are `codescanning`, `secretscanning`, `dependabot` or simply `all`. Default value: `all`.
-- `SECRET_TYPE_FILTER`: A comma-separated list of secret types to filter secret scanning alerts. For example: `password,api_key,oauth_token`. If not set, all secret types are included.
 
 The first two are only needed if you're running this in a GitHub Enterprise Server or GitHub AE environment.  The last one is useful if you only want to get data on a specific feature.  For example, if you only want to get data on secret scanning, you can set `FEATURES` to `secretscanning`. Here's just another example how you would configure this on a GitHub Enterprise Server:
 

main.py

@@ -34,7 +34,6 @@
 report_scope = os.getenv("GITHUB_REPORT_SCOPE", "repository")
 scope_name = os.getenv("SCOPE_NAME", os.getenv("GITHUB_REPOSITORY"))
 requested_features = os.getenv("FEATURES")
-secret_type_filter = os.getenv("SECRET_TYPE_FILTER")
 if (requested_features is None) or (requested_features == "all"):
     features = FEATURES
 else:
@@ -53,7 +52,7 @@
         # secret scanning
         if "secretscanning" in features:
             try:
-                secrets_list = secret_scanning.get_enterprise_ss_alerts(api_endpoint, github_pat, scope_name, secret_type_filter)
+                secrets_list = secret_scanning.get_enterprise_ss_alerts(api_endpoint, github_pat, scope_name)
                 secret_scanning.write_enterprise_ss_list(secrets_list)
             except Exception as e:
                 if any(x in str(e).lower() for x in secret_scanning_disabled_strings):
@@ -105,7 +104,7 @@
         # secret scanning
         if "secretscanning" in features:
             try:
-                secrets_list = secret_scanning.get_org_ss_alerts(api_endpoint, github_pat, scope_name, secret_type_filter)
+                secrets_list = secret_scanning.get_org_ss_alerts(api_endpoint, github_pat, scope_name)
                 secret_scanning.write_org_ss_list(secrets_list)
             except Exception as e:
                 if any(x in str(e).lower() for x in secret_scanning_disabled_strings):
@@ -133,7 +132,7 @@
         # secret scanning
         if "secretscanning" in features:
             try:
-                secrets_list = secret_scanning.get_repo_ss_alerts(api_endpoint, github_pat, scope_name, secret_type_filter)
+                secrets_list = secret_scanning.get_repo_ss_alerts(api_endpoint, github_pat, scope_name)
                 secret_scanning.write_repo_ss_list(secrets_list)
             except Exception as e:
                 if any(x in str(e).lower() for x in secret_scanning_disabled_strings):

src/secret_scanning.py

@@ -5,25 +5,51 @@
 from . import api_helpers
 
 
-def get_repo_ss_alerts(api_endpoint, github_pat, repo_name, secret_type_filter=None):
+def get_repo_ss_alerts(api_endpoint, github_pat, repo_name):
     """
     Get all the secret scanning alerts on a given repository.
 
     Inputs:
     - API endpoint (for GHES/GHAE compatibility)
     - PAT of appropriate scope
     - Repository name
-    - Secret type filter (optional comma-separated list of secret types)
 
     Outputs:
-    - List of _all_ secret scanning alerts on the repository
+    - List of _all_ secret scanning alerts on the repository (both default and generic secret types)
     """
-    url = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&page=1"
-    if secret_type_filter:
-        url += f"&secret_type={secret_type_filter}"
-    ss_alerts = api_helpers.make_api_call(url, github_pat)
-    print(f"Found {len(ss_alerts)} secret scanning alerts in {repo_name}")
-    return ss_alerts
+    # First call: get default secret types (without any filters)
+    url_default = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&page=1"
+    ss_alerts_default = api_helpers.make_api_call(url_default, github_pat)
+    
+    # Second call: get generic secret types with hardcoded list
+    generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key"
+    url_generic = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&page=1&secret_type={generic_secret_types}"
+    ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat)
+    
+    # Combine results and deduplicate
+    combined_alerts = []
+    alert_numbers_seen = set()
+    duplicates_found = False
+    
+    # Add default alerts
+    for alert in ss_alerts_default:
+        alert_numbers_seen.add(alert["number"])
+        combined_alerts.append(alert)
+    
+    # Add generic alerts, checking for duplicates
+    for alert in ss_alerts_generic:
+        if alert["number"] in alert_numbers_seen:
+            duplicates_found = True
+        else:
+            alert_numbers_seen.add(alert["number"])
+            combined_alerts.append(alert)
+    
+    # Warn if duplicates were found
+    if duplicates_found:
+        print(f"::warning::Duplicate secret scanning alerts detected in {repo_name}. Please report this behavior via an issue to the repository owners as the API behavior may have changed.")
+    
+    print(f"Found {len(combined_alerts)} secret scanning alerts in {repo_name} ({len(ss_alerts_default)} default, {len(ss_alerts_generic)} generic)")
+    return combined_alerts
 
 
 def write_repo_ss_list(secrets_list):
@@ -72,25 +98,51 @@ def write_repo_ss_list(secrets_list):
             )
 
 
-def get_org_ss_alerts(api_endpoint, github_pat, org_name, secret_type_filter=None):
+def get_org_ss_alerts(api_endpoint, github_pat, org_name):
     """
     Get all the secret scanning alerts on a given organization.
 
     Inputs:
     - API endpoint (for GHES/GHAE compatibility)
     - PAT of appropriate scope
     - Organization name
-    - Secret type filter (optional comma-separated list of secret types)
 
     Outputs:
-    - List of _all_ secret scanning alerts on the organization
+    - List of _all_ secret scanning alerts on the organization (both default and generic secret types)
     """
-    url = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&page=1"
-    if secret_type_filter:
-        url += f"&secret_type={secret_type_filter}"
-    ss_alerts = api_helpers.make_api_call(url, github_pat)
-    print(f"Found {len(ss_alerts)} secret scanning alerts in {org_name}")
-    return ss_alerts
+    # First call: get default secret types (without any filters)
+    url_default = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&page=1"
+    ss_alerts_default = api_helpers.make_api_call(url_default, github_pat)
+    
+    # Second call: get generic secret types with hardcoded list
+    generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key"
+    url_generic = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&page=1&secret_type={generic_secret_types}"
+    ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat)
+    
+    # Combine results and deduplicate
+    combined_alerts = []
+    alert_numbers_seen = set()
+    duplicates_found = False
+    
+    # Add default alerts
+    for alert in ss_alerts_default:
+        alert_numbers_seen.add(alert["number"])
+        combined_alerts.append(alert)
+    
+    # Add generic alerts, checking for duplicates
+    for alert in ss_alerts_generic:
+        if alert["number"] in alert_numbers_seen:
+            duplicates_found = True
+        else:
+            alert_numbers_seen.add(alert["number"])
+            combined_alerts.append(alert)
+    
+    # Warn if duplicates were found
+    if duplicates_found:
+        print(f"::warning::Duplicate secret scanning alerts detected in {org_name}. Please report this behavior via an issue to the repository owners as the API behavior may have changed.")
+    
+    print(f"Found {len(combined_alerts)} secret scanning alerts in {org_name} ({len(ss_alerts_default)} default, {len(ss_alerts_generic)} generic)")
+    return combined_alerts
 
 
 def write_org_ss_list(secrets_list):
@@ -153,7 +205,7 @@ def write_org_ss_list(secrets_list):
             )
 
 
-def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug, secret_type_filter=None):
+def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug):
     """
     Get all the secret scanning alerts on a given enterprise.
 
@@ -162,17 +214,43 @@ def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug, secret_t
     - PAT of appropriate scope
     - Enterprise slug (enterprise name URL, documented below)
         - https://docs.github.com/en/rest/reference/enterprise-admin
-    - Secret type filter (optional comma-separated list of secret types)
 
     Outputs:
-    - List of _all_ secret scanning alerts on the enterprise
+    - List of _all_ secret scanning alerts on the enterprise (both default and generic secret types)
     """
-    url = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&page=1"
-    if secret_type_filter:
-        url += f"&secret_type={secret_type_filter}"
-    ss_alerts = api_helpers.make_api_call(url, github_pat)
-    print(f"Found {len(ss_alerts)} secret scanning alerts in {enterprise_slug}")
-    return ss_alerts
+    # First call: get default secret types (without any filters)
+    url_default = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&page=1"
+    ss_alerts_default = api_helpers.make_api_call(url_default, github_pat)
+    
+    # Second call: get generic secret types with hardcoded list
+    generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key"
+    url_generic = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&page=1&secret_type={generic_secret_types}"
+    ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat)
+    
+    # Combine results and deduplicate
+    combined_alerts = []
+    alert_numbers_seen = set()
+    duplicates_found = False
+    
+    # Add default alerts
+    for alert in ss_alerts_default:
+        alert_numbers_seen.add(alert["number"])
+        combined_alerts.append(alert)
+    
+    # Add generic alerts, checking for duplicates
+    for alert in ss_alerts_generic:
+        if alert["number"] in alert_numbers_seen:
+            duplicates_found = True
+        else:
+            alert_numbers_seen.add(alert["number"])
+            combined_alerts.append(alert)
+    
+    # Warn if duplicates were found
+    if duplicates_found:
+        print(f"::warning::Duplicate secret scanning alerts detected in {enterprise_slug}. Please report this behavior via an issue to the repository owners as the API behavior may have changed.")
+    
+    print(f"Found {len(combined_alerts)} secret scanning alerts in {enterprise_slug} ({len(ss_alerts_default)} default, {len(ss_alerts_generic)} generic)")
+    return combined_alerts
 
 
 def write_enterprise_ss_list(secrets_list):