feat: require compliance with the black python code formatter to pass…

… lint checks (#481)

Makefile

@@ -9,13 +9,19 @@ deps:
 deps-update:
 	pipenv update
 
-lint:
+lint: lint-pylint lint-fmt
+
+lint-pylint:
 	pipenv run bandit -r $(dirs) --skip B101  # allow assert statements in tests
 	pipenv run pylint $(dirs) \
 	  --disable=missing-docstring,duplicate-code,import-error,fixme,consider-iterating-dictionary,global-variable-not-assigned \
 	  --load-plugins=pylint.extensions.mccabe \
 	  --max-line-length=100
 
+lint-fmt:
+	@echo Checking python file formatting with the black code style checker
+	pipenv run black --line-length=100 --check $(dirs)
+
 venv:
 	pipenv install --dev
 

aws_cloudtrail_rules/aws_ami_modified_for_public_access.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_cloudtrail_created.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 # API calls that are indicative of CloudTrail changes
 CLOUDTRAIL_CREATE_UPDATE = {

aws_cloudtrail_rules/aws_cloudtrail_stopped.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success, lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 # API calls that are indicative of CloudTrail changes
 CLOUDTRAIL_STOP_DELETE = {

aws_cloudtrail_rules/aws_codebuild_made_public.py

@@ -1,5 +1,5 @@
 from panther import lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_console_login_failed.py

@@ -1,5 +1,5 @@
 from panther import lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_console_login_without_mfa.py

@@ -2,7 +2,7 @@
 import logging
 
 from panther import lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 from panther_oss_helpers import check_account_age
 
 # Set to True for environments that permit direct role assumption via external IDP

aws_cloudtrail_rules/aws_console_login_without_saml.py

@@ -1,5 +1,5 @@
 from panther import lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_console_root_login_failed.py

@@ -1,5 +1,5 @@
 from panther import lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_ec2_manual_security_group_changes.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, pattern_match_list, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get, pattern_match_list
 
 PROD_ACCOUNT_IDS = {"11111111111111", "112233445566"}
 SG_CHANGE_EVENTS = {

aws_cloudtrail_rules/aws_iam_assume_role_blocklist_ignored.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 # This is a list of role ARNs that should not be assumed by users in normal operations
 ASSUME_ROLE_BLOCKLIST = [

aws_cloudtrail_rules/aws_iam_entity_created_without_cloudformation.py

@@ -1,7 +1,7 @@
 import re
 
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 # The role dedicated for IAM administration
 IAM_ADMIN_ROLES = {

aws_cloudtrail_rules/aws_iam_user_recon_denied.py

@@ -1,7 +1,7 @@
 from ipaddress import ip_address
 
 from panther import lookup_aws_account_name
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 # service/event patterns to monitor
 RECON_ACTIONS = {

aws_cloudtrail_rules/aws_key_compromised.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 EXPOSED_CRED_POLICY = "AWSExposedCredentialPolicy_DO_NOT_REMOVE"
 

aws_cloudtrail_rules/aws_network_acl_permissive_entry.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_resource_made_public.py

@@ -1,7 +1,7 @@
 import json
 
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 from policyuniverse.policy import Policy
 
 

aws_cloudtrail_rules/aws_root_access_key_created.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_root_console_login.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_root_failed_console_login.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_root_password_changed.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_s3_activity_greynoise.py

@@ -3,7 +3,6 @@
 from panther_base_helpers import deep_get, pattern_match_list
 from panther_greynoise_helpers import GetGreyNoiseObject, GetGreyNoiseRiotObject
 
-
 # pylint: disable=too-many-return-statements,invalid-name,unused-argument,global-at-module-level,global-variable-undefined
 
 # Monitor for GetObject events from S3.
@@ -57,8 +56,8 @@ def rule(event):
         # Filter: Roles that generate FP's if used from AWS IP Space
         if pattern_match_list(deep_get(event, "userIdentity", "arn"), _ALLOWED_ROLES):
             # Only Greynoise advanced provides AS organization info
-            if NOISE.subscription_level() == 'advanced':
-                if NOISE.organization() == 'Amazon.com, Inc.':
+            if NOISE.subscription_level() == "advanced":
+                if NOISE.organization() == "Amazon.com, Inc.":
                     return False
             # return false if the role is seen and we are not able to valide the AS organization
             else:

aws_cloudtrail_rules/aws_s3_bucket_deleted.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_s3_bucket_policy_modified.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 # API calls that are indicative of KMS CMK Deletion
 S3_POLICY_CHANGE_EVENTS = {

aws_cloudtrail_rules/aws_security_configuration_change.py

@@ -1,7 +1,7 @@
 from fnmatch import fnmatch
 
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 SECURITY_CONFIG_ACTIONS = {
     "DeleteAccountPublicAccessBlock",

aws_cloudtrail_rules/aws_snapshot_made_public.py

@@ -1,7 +1,7 @@
 from collections.abc import Mapping
 
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 
 def rule(event):

aws_cloudtrail_rules/aws_unauthorized_api_call.py

@@ -1,6 +1,6 @@
 from ipaddress import ip_address
 
-from panther_base_helpers import aws_strip_role_session_id, deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, aws_strip_role_session_id, deep_get
 
 # Do not alert on these access denied errors for these events.
 # Events could be exceptions because they are particularly noisy and provide little to no value,

aws_cloudtrail_rules/aws_update_credentials.py

@@ -1,5 +1,5 @@
 from panther import aws_cloudtrail_success
-from panther_base_helpers import deep_get, aws_rule_context
+from panther_base_helpers import aws_rule_context, deep_get
 
 UPDATE_EVENTS = {"ChangePassword", "CreateAccessKey", "CreateLoginProfile", "CreateUser"}
 

aws_guardduty_rules/aws_guardduty_high_sev_findings.py

@@ -1,4 +1,6 @@
 from panther_base_helpers import aws_rule_context
+
+
 def rule(event):
     return 7.0 <= float(event.get("severity", 0)) <= 8.9
 

aws_guardduty_rules/aws_guardduty_low_sev_findings.py

@@ -1,4 +1,6 @@
 from panther_base_helpers import aws_rule_context
+
+
 def rule(event):
     return 0.1 <= float(event.get("severity", 0)) <= 3.9
 

aws_guardduty_rules/aws_guardduty_med_sev_findings.py

@@ -1,4 +1,6 @@
 from panther_base_helpers import aws_rule_context
+
+
 def rule(event):
     return 4.0 <= float(event.get("severity", 0)) <= 6.9
 

aws_s3_rules/aws_s3_access_error.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import pattern_match, aws_rule_context
+from panther_base_helpers import aws_rule_context, pattern_match
 
 # https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 HTTP_STATUS_CODES_TO_MONITOR = {

aws_s3_rules/aws_s3_access_ip_allowlist.py

@@ -1,4 +1,5 @@
 from ipaddress import ip_network
+
 from panther_base_helpers import aws_rule_context
 
 BUCKETS_TO_MONITOR = {

aws_s3_rules/aws_s3_insecure_access.py

@@ -1,4 +1,4 @@
-from panther_base_helpers import pattern_match, aws_rule_context
+from panther_base_helpers import aws_rule_context, pattern_match
 
 
 def rule(event):

aws_s3_rules/aws_s3_unauthenticated_access.py

@@ -1,4 +1,5 @@
 from panther_base_helpers import aws_rule_context
+
 # A list of buckets where authenticated access is expected
 AUTH_BUCKETS = {"example-bucket"}
 

aws_s3_rules/aws_s3_unknown_requester_get_object.py

@@ -1,4 +1,5 @@
 from fnmatch import fnmatch
+
 from panther_base_helpers import aws_rule_context
 
 # pylint: disable=line-too-long

aws_vpc_flow_rules/aws_vpc_healthy_log_status.py

@@ -1,4 +1,6 @@
 from panther_base_helpers import aws_rule_context
+
+
 def rule(event):
     return event.get("log-status") == "SKIPDATA"
 

aws_vpc_flow_rules/aws_vpc_inbound_traffic_port_allowlist.py

@@ -1,4 +1,5 @@
 from ipaddress import ip_network
+
 from panther_base_helpers import aws_rule_context
 
 APPROVED_PORTS = {

aws_vpc_flow_rules/aws_vpc_inbound_traffic_port_blocklist.py

@@ -1,4 +1,5 @@
 from ipaddress import ip_network
+
 from panther_base_helpers import aws_rule_context
 
 CONTROLLED_PORTS = {

aws_vpc_flow_rules/aws_vpc_unapproved_outbound_dns.py

@@ -1,4 +1,5 @@
 from ipaddress import ip_network
+
 from panther_base_helpers import aws_rule_context
 
 APPROVED_DNS_SERVERS = {

data_models/gcp_data_model.py

@@ -2,8 +2,8 @@
 from fnmatch import fnmatch
 
 import panther_event_type_helpers as event_type
-from panther_base_helpers import get_binding_deltas
 from panther_analysis_tool.enriched_event import PantherEvent
+from panther_base_helpers import get_binding_deltas
 
 ADMIN_ROLES = {
     # Primitive Rolesx

gcp_audit_rules/gcp_iam_org_folder_changes.py

@@ -1,31 +1,38 @@
 from panther_base_helpers import deep_get
 
+
 def rule(event):
     # Return True to match the log event and trigger an alert.
     logname = deep_get(event, "logName")
-    return deep_get(event, "protoPayload", "methodName") == "SetIamPolicy" and \
-            (logname.startswith("organizations") or \
-            logname.startswith("folder") ) and \
-            logname.endswith("/logs/cloudaudit.googleapis.com%2Factivity")
+    return (
+        deep_get(event, "protoPayload", "methodName") == "SetIamPolicy"
+        and (logname.startswith("organizations") or logname.startswith("folder"))
+        and logname.endswith("/logs/cloudaudit.googleapis.com%2Factivity")
+    )
+
 
 def title(event):
     # use unified data model field in title
     return (
         f"{event.get('p_log_type')}: [{event.udm('actor_user')}] made manual changes to Org policy"
     )
 
+
 def alert_context(event):
     return {
         "actor": event.udm("actor_user"),
-        "policy_change": deep_get(event,  "protoPayload", "serviceData", "policyDelta"),
+        "policy_change": deep_get(event, "protoPayload", "serviceData", "policyDelta"),
         "caller_ip": deep_get(event, "protoPayload", "requestMetadata", "callerIP"),
-        "user_agent": deep_get(event, "protoPayload", "requestMetadata", "callerSuppliedUserAgent")
+        "user_agent": deep_get(event, "protoPayload", "requestMetadata", "callerSuppliedUserAgent"),
     }
 
+
 def severity(event):
-    if deep_get(event,
-                "protoPayload",
-                "requestMetadata",
-                "callerSuppliedUserAgent").lower().find('terraform') != -1:
-        return 'INFO'
-    return 'HIGH'
+    if (
+        deep_get(event, "protoPayload", "requestMetadata", "callerSuppliedUserAgent")
+        .lower()
+        .find("terraform")
+        != -1
+    ):
+        return "INFO"
+    return "HIGH"

global_helpers/panther_base_helpers.py

@@ -226,7 +226,7 @@ def slack_alert_context(event: dict):
         "actor-name": deep_get(event, "actor", "user", "name", default="<MISSING_NAME>"),
         "actor-email": deep_get(event, "actor", "user", "email", default="<MISSING_EMAIL>"),
         "actor-ip": deep_get(event, "context", "ip_address", default="<MISSING_IP>"),
-        "user-agent": deep_get(event, "context", "ua", default="<MISSING_UA>")
+        "user-agent": deep_get(event, "context", "ua", default="<MISSING_UA>"),
     }
 
 

global_helpers/panther_cloudflare_helpers.py

@@ -18,7 +18,7 @@
     "botManagement": "Bot Management",
     "dlp": "Data Loss Prevention",
     "firewallManaged": "Firewall Managed Rules",
-    "firewallCustom": "Firewall Custom Rulesets"
+    "firewallCustom": "Firewall Custom Rulesets",
 }