Merge pull request #4607 from mozilla/add-ruff-again-mpp-79

MPP-79: Add `ruff` Python linter

.circleci/config.yml

@@ -378,7 +378,7 @@ jobs:
         description: "What command should the job run?"
         default: "pytest"
         type: enum
-        enum: ["pytest", "black", "mypy", "build_email_tracker_lists", "build_glean", "check_glean"]
+        enum: ["pytest", "black", "mypy", "build_email_tracker_lists", "build_glean", "check_glean", "ruff"]
       test_results_filename:
         description: "What is the name of the jUnit XML test output? (Optional)"
         default: ""
@@ -494,6 +494,7 @@ jobs:
             echo "export TEST_DB_NAME=$(printf '%q' "${TEST_DB_NAME}")" >> "$TMP_ENV"
             echo "export TEST_DB_URL=$(printf '%q' "${TEST_DB_URL}")" >> "$TMP_ENV"
             echo "export DATABASE_ENGINE=$(printf '%q' "${DATABASE_ENGINE}")" >> "$TMP_ENV"
+            echo "export TEST_RESULTS_DIR=job-results" >> "$TMP_ENV"
 
             cat "$TMP_ENV" | tee --append "$BASH_ENV"
             cat /home/circleci/project/.circleci/python_job.bash >> "$BASH_ENV"
@@ -504,7 +505,7 @@ jobs:
           steps:
             - run:
                 name: Create job-results directory
-                command: mkdir job-results
+                command: mkdir -p "$TEST_RESULTS_DIR"
       - run:
           name: Set test defaults
           command: cp .env-dist .env
@@ -608,6 +609,13 @@ workflows:
           command: black
           filters: *default_filters
 
+      - python_job:
+          name: ruff linting check
+          command: ruff
+          test_results_filename: "ruff.xml"
+          allow_fail: true
+          filters: *default_filters
+
       - python_job:
           name: mypy type check
           command: mypy

.circleci/python_job.bash

@@ -8,6 +8,7 @@
 : ${PHONES_ENABLED:=0}
 : ${PYTEST_FAIL_FAST:=0}
 : ${PYTEST_MIGRATIONS_MODE:=0}
+: ${TEST_RESULTS_DIR:=tmp}
 : ${TEST_RESULTS_FILENAME:=}
 : ${DATABASE_URL:=sqlite:///db.sqlite3}
 : ${TEST_DB_NAME:=test.sqlite3}
@@ -28,7 +29,7 @@ function run_mypy {
     if [ $MYPY_STRICT -ne 0 ]; then MYPY_ARGS+=("--strict"); fi
     if [ -n "$TEST_RESULTS_FILENAME" ]
     then
-        MYPY_ARGS+=("--junit-xml" "job-results/${TEST_RESULTS_FILENAME}")
+        MYPY_ARGS+=("--junit-xml" "${TEST_RESULTS_DIR}/${TEST_RESULTS_FILENAME}")
     fi
     MYPY_ARGS+=(".")
 
@@ -56,7 +57,7 @@ function run_pytest {
     if [ $CREATE_DB -ne 0 ]; then PYTEST_ARGS+=("--create-db"); fi
     if [ -n "$TEST_RESULTS_FILENAME" ] && [ $SKIP_RESULTS != 1 ]
     then
-        PYTEST_ARGS+=("--junit-xml=job-results/$TEST_RESULTS_FILENAME")
+        PYTEST_ARGS+=("--junit-xml=${TEST_RESULTS_DIR}/$TEST_RESULTS_FILENAME")
     fi
     PYTEST_ARGS+=(".")
 
@@ -84,6 +85,7 @@ function run_build_glean {
     set -x
     glean_parser translate --format python_server --output ${OUTPUT_FOLDER} ${INPUT_YAML}
     black ${OUTPUT_FOLDER}
+    ruff check --fix --ignore E501 ${OUTPUT_FOLDER}
     { set +x; } 2>/dev/null
     case "$OSTYPE" in
         darwin* | bsd*)
@@ -169,14 +171,28 @@ function run_check_glean {
     fi
 }
 
+# Run ruff to lint python code
+function run_ruff {
+    if [ -n "$TEST_RESULTS_FILENAME" ]
+    then
+        # Run with output to a test results file instead of stdout
+        set -x
+        ruff check --exit-zero --output-format junit --output-file "${TEST_RESULTS_DIR}/${TEST_RESULTS_FILENAME}" .
+        { set +x; } 2>/dev/null
+    fi
+
+    set -x
+    ruff check .
+}
+
 
 # Run a command by name
 # $1 - The command to run - black, mypy, pytest, or build_email_tracker_lists
 # Remaining arguments are passed to the run_COMMAND function
 function run_command {
     local COMMAND=${1:-}
     case $COMMAND in
-        black | mypy | pytest | build_email_tracker_lists | build_glean | check_glean)
+        black | mypy | pytest | build_email_tracker_lists | build_glean | check_glean | ruff)
             :;;
         "")
             echo "No command passed - '$COMMAND'"

.lintstagedrc.js

@@ -6,5 +6,5 @@ module.exports = {
       .map((file) => file.split(process.cwd())[1])
       .join(" --file ")}`,
   "*.md": "prettier --write",
-  "*.py": ["black", "mypy"],
+  "*.py": ["black", "mypy", "ruff check --fix"],
 }

api/authentication.py

@@ -1,13 +1,12 @@
-from datetime import datetime, timezone
-from typing import Any
 import logging
 import shlex
-
-import requests
+from datetime import UTC, datetime
+from typing import Any
 
 from django.conf import settings
 from django.core.cache import cache
 
+import requests
 from allauth.socialaccount.models import SocialAccount
 from rest_framework.authentication import BaseAuthentication, get_authorization_header
 from rest_framework.exceptions import (
@@ -18,7 +17,6 @@
     PermissionDenied,
 )
 
-
 logger = logging.getLogger("events")
 INTROSPECT_TOKEN_URL = (
     "%s/introspect" % settings.SOCIALACCOUNT_PROVIDERS["fxa"]["OAUTH_ENDPOINT"]
@@ -95,10 +93,10 @@ def get_fxa_uid_from_oauth_token(token: str, use_cache=True) -> str:
 
     # cache valid access_token and fxa_resp_data until access_token expiration
     # TODO: revisit this since the token can expire before its time
-    if type(fxa_resp_data.get("json", {}).get("exp")) is int:
+    if isinstance(fxa_resp_data.get("json", {}).get("exp"), int):
         # Note: FXA iat and exp are timestamps in *milliseconds*
         fxa_token_exp_time = int(fxa_resp_data["json"]["exp"] / 1000)
-        now_time = int(datetime.now(timezone.utc).timestamp())
+        now_time = int(datetime.now(UTC).timestamp())
         fxa_token_exp_cache_timeout = fxa_token_exp_time - now_time
         if fxa_token_exp_cache_timeout > cache_timeout:
             # cache until access_token expires (matched Relay user)

api/permissions.py

@@ -1,9 +1,7 @@
 from rest_framework import permissions
-
-from emails.models import Profile
-
 from waffle import flag_is_active
 
+from emails.models import Profile
 
 READ_METHODS = ["GET", "HEAD"]
 

api/schema.py

@@ -1,6 +1,6 @@
 """Schema Extensions for drf-spectacular"""
 
-from typing import Callable
+from collections.abc import Callable
 
 from drf_spectacular.extensions import OpenApiAuthenticationExtension
 from drf_spectacular.openapi import AutoSchema

api/serializers/__init__.py

@@ -1,7 +1,7 @@
 from django.contrib.auth.models import User
 from django.db.models import prefetch_related_objects
 
-from rest_framework import serializers, exceptions
+from rest_framework import exceptions, serializers
 from waffle import get_waffle_flag_model
 
 from emails.models import DomainAddress, Profile, RelayAddress

api/tests/authentication_tests.py

@@ -1,28 +1,22 @@
 from datetime import datetime
 
-from model_bakery import baker
-import responses
-
 from django.core.cache import cache
 from django.test import RequestFactory, TestCase
 
+import responses
 from allauth.socialaccount.models import SocialAccount
-from rest_framework.exceptions import (
-    APIException,
-    AuthenticationFailed,
-    NotFound,
-)
+from model_bakery import baker
+from rest_framework.exceptions import APIException, AuthenticationFailed, NotFound
 from rest_framework.test import APIClient
 
 from ..authentication import (
+    INTROSPECT_TOKEN_URL,
     FxaTokenAuthentication,
     get_cache_key,
     get_fxa_uid_from_oauth_token,
     introspect_token,
-    INTROSPECT_TOKEN_URL,
 )
 
-
 MOCK_BASE = "api.authentication"
 
 

api/tests/iq_views_tests.py

@@ -1,28 +1,24 @@
-from allauth.socialaccount.models import SocialAccount
-import pytest
+from unittest.mock import Mock, patch
 
 from django.conf import settings
 
-pytestmark = pytest.mark.skipif(
-    not settings.PHONES_ENABLED, reason="PHONES_ENABLED is False"
-)
-pytestmark = pytest.mark.skipif(not settings.IQ_ENABLED, reason="IQ_ENABLED is False")
-
+import pytest
 import responses
-from unittest.mock import Mock, patch
-
-from twilio.rest import Client
-
-
+from allauth.socialaccount.models import SocialAccount
 from rest_framework.test import RequestsClient
+from twilio.rest import Client
 
 if settings.PHONES_ENABLED:
     from api.views.phones import compute_iq_mac
     from phones.models import InboundContact, iq_fmt
 
-from phones.tests.models_tests import make_phone_test_user
 from api.tests.phones_views_tests import _make_real_phone, _make_relay_number
+from phones.tests.models_tests import make_phone_test_user
 
+pytestmark = pytest.mark.skipif(
+    not settings.PHONES_ENABLED, reason="PHONES_ENABLED is False"
+)
+pytestmark = pytest.mark.skipif(not settings.IQ_ENABLED, reason="IQ_ENABLED is False")
 
 API_ROOT = "http://127.0.0.1:8000"
 INBOUND_SMS_PATH = f"{API_ROOT}/api/v1/inbound_sms_iq/"

api/tests/phones_views_tests.py

@@ -1,28 +1,26 @@
+import re
+from collections.abc import Iterator
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Literal
-from collections.abc import Iterator
-from unittest.mock import Mock, patch, call
-import re
-
-from twilio.request_validator import RequestValidator
-from twilio.rest import Client
+from unittest.mock import Mock, call, patch
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.test.utils import override_settings
 
+import pytest
 from model_bakery import baker
 from rest_framework.test import APIClient
 from twilio.base.exceptions import TwilioRestException
+from twilio.request_validator import RequestValidator
+from twilio.rest import Client
 from waffle.testutils import override_flag
-import pytest
 
 from emails.models import Profile
 
-
 if settings.PHONES_ENABLED:
-    from api.views.phones import _match_by_prefix, MatchByPrefix
+    from api.views.phones import MatchByPrefix, _match_by_prefix
     from phones.models import InboundContact, RealPhone, RelayNumber
     from phones.tests.models_tests import make_phone_test_user
 
@@ -98,55 +96,55 @@ def user_with_sms_activity(outbound_phone_user, mocked_twilio_client):
     InboundContact.objects.create(
         relay_number=relay_number,
         inbound_number="+13015550001",
-        last_inbound_date=datetime(2023, 3, 1, 12, 5, tzinfo=timezone.utc),
+        last_inbound_date=datetime(2023, 3, 1, 12, 5, tzinfo=UTC),
         last_inbound_type="text",
-        last_text_date=datetime(2023, 3, 1, 12, 5, tzinfo=timezone.utc),
+        last_text_date=datetime(2023, 3, 1, 12, 5, tzinfo=UTC),
     )
     # Second SMS contact
     InboundContact.objects.create(
         relay_number=relay_number,
         inbound_number="+13015550002",
-        last_inbound_date=datetime(2023, 3, 2, 13, 5, tzinfo=timezone.utc),
+        last_inbound_date=datetime(2023, 3, 2, 13, 5, tzinfo=UTC),
         last_inbound_type="text",
-        last_text_date=datetime(2023, 3, 2, 13, 5, tzinfo=timezone.utc),
+        last_text_date=datetime(2023, 3, 2, 13, 5, tzinfo=UTC),
     )
     # Voice contact
     InboundContact.objects.create(
         relay_number=relay_number,
         inbound_number="+13015550003",
-        last_inbound_date=datetime(2023, 3, 3, 8, 30, tzinfo=timezone.utc),
+        last_inbound_date=datetime(2023, 3, 3, 8, 30, tzinfo=UTC),
         last_inbound_type="call",
-        last_call_date=datetime(2023, 3, 3, 8, 30, tzinfo=timezone.utc),
+        last_call_date=datetime(2023, 3, 3, 8, 30, tzinfo=UTC),
     )
     twilio_messages = [
         MockTwilioMessage(
             from_="+13015550001",
             to=relay_number.number,
-            date_sent=datetime(2023, 3, 1, 12, 0, tzinfo=timezone.utc),
+            date_sent=datetime(2023, 3, 1, 12, 0, tzinfo=UTC),
             body="Send Y to confirm appointment",
         ),
         MockTwilioMessage(
             from_=relay_number.number,
             to="+13015550001",
-            date_sent=datetime(2023, 3, 1, 12, 5, tzinfo=timezone.utc),
+            date_sent=datetime(2023, 3, 1, 12, 5, tzinfo=UTC),
             body="Y",
         ),
         MockTwilioMessage(
             from_="+13015550002",
             to=relay_number.number,
-            date_sent=datetime(2023, 3, 2, 13, 0, tzinfo=timezone.utc),
+            date_sent=datetime(2023, 3, 2, 13, 0, tzinfo=UTC),
             body="Donate $100 to Senator Smith?",
         ),
         MockTwilioMessage(
             from_=relay_number.number,
             to="+13015550002",
-            date_sent=datetime(2023, 3, 2, 13, 5, tzinfo=timezone.utc),
+            date_sent=datetime(2023, 3, 2, 13, 5, tzinfo=UTC),
             body="STOP STOP STOP",
         ),
         MockTwilioMessage(
             from_=relay_number.number,
             to="+13015550004",
-            date_sent=datetime(2023, 3, 4, 20, 55, tzinfo=timezone.utc),
+            date_sent=datetime(2023, 3, 4, 20, 55, tzinfo=UTC),
             body="U Up?",
         ),
     ]