chore: upgrade python to 3.11 (#23206)

.github/actions/run-backend-tests/action.yml

@@ -6,7 +6,7 @@ name: Run Django tests
 inputs:
  python-version:
  required: true
- description: Python version, e.g. 3.10.10
+ description: Python version, e.g. 3.11.9
  clickhouse-server-image:
  required: true
  description: ClickHouse server image tag, e.g. clickhouse/clickhouse-server:latest

.github/workflows/benchmark.yml

@@ -54,7 +54,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/build-hogql-parser.yml

@@ -73,7 +73,7 @@ jobs:
  - if: ${{ !endsWith(matrix.os, '-arm') }}
  uses: actions/setup-python@v4
  with:
- python-version: '3.10'
+ python-version: '3.11'
 
  # Compiling Python 3.11 from source on ARM. We tried using the "deadsnakes" ARM repo, but it was flakey.
  - if: ${{ endsWith(matrix.os, '-arm') }}

.github/workflows/ci-backend-update-test-timing.yml

@@ -28,7 +28,7 @@ jobs:
  concurrency: 1
  group: 1
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- python-version: '3.10.10'
+ python-version: '3.11.9'
  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
  segment: 'FOSS'
  person-on-events: false

.github/workflows/ci-backend.yml

@@ -108,7 +108,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -163,7 +163,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -232,7 +232,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ['3.10.10']
+ python-version: ['3.11.9']
  clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
  segment: ['Core']
  person-on-events: [false, true]
@@ -243,7 +243,7 @@ jobs:
  - segment: 'Temporal'
  person-on-events: false
  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
- python-version: '3.10.10'
+ python-version: '3.11.9'
  concurrency: 1
  group: 1
 
@@ -331,7 +331,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-hog.yml

@@ -70,7 +70,7 @@ jobs:
  if: needs.changes.outputs.hog == 'true'
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-plugin-server.yml

@@ -115,7 +115,7 @@ jobs:
  if: needs.changes.outputs.plugin-server == 'true'
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -207,7 +207,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

bin/build-schema-python.sh

@@ -4,25 +4,27 @@ set -e
 
 # Generate schema.py from schema.json
 datamodel-codegen \
- --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp \
+ --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.11 --disable-timestamp \
  --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum \
  --input frontend/src/queries/schema.json --input-file-type jsonschema \
  --output posthog/schema.py --output-model-type pydantic_v2.BaseModel \
  --custom-file-header "# mypy: disable-error-code=\"assignment\"" \
  --set-default-enum-member --capitalise-enum-members \
  --wrap-string-literal
+
 # Format schema.py
 ruff format posthog/schema.py
+
 # Check schema.py and autofix
 ruff check --fix posthog/schema.py
-# HACK: Datamodel-codegen output for enum-type fields with a default is invalid – the default value is a plain string,
-# and not the expected enum member. We fix this using sed, which is pretty hacky, but does the job.
-# Specifically, we need to replace `Optional[PropertyOperator] = "exact"`
-# with `Optional[PropertyOperator] = PropertyOperator("exact")` to make the default value valid.
-# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1929 is resolved.
+
+# Replace class Foo(str, Enum) with class Foo(StrEnum) for proper handling in format strings in python 3.11
+# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1313 is resolved
 if [[ "$OSTYPE" == "darwin"* ]]; then
  # sed needs `-i` to be followed by `''` on macOS
- sed -i '' -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
+ sed -i '' -e 's/str, Enum/StrEnum/g' posthog/schema.py
+ sed -i '' 's/from enum import Enum/from enum import Enum, StrEnum/g' posthog/schema.py
 else
- sed -i -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
-fi
+ sed -i -e 's/str, Enum/StrEnum/g' posthog/schema.py
+ sed -i 's/from enum import Enum/from enum import Enum, StrEnum/g' posthog/schema.py
+fi

ee/api/test/__snapshots__/test_time_to_see_data.ambr

@@ -20,7 +20,7 @@
  "first_name": "",
  "last_name": "",
  "email": "",
- "is_email_verified": false
+ "is_email_verified": null
  }
  },
  "children": [

ee/clickhouse/views/test/test_clickhouse_experiments.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, UTC
 from django.core.cache import cache
 from flaky import flaky
 from rest_framework import status
@@ -1601,8 +1601,8 @@ def test_create_exposure_cohort_for_experiment_with_custom_action_filters_exposu
  explicit_datetime = parser.isoparse(target_filter["explicit_datetime"])
 
  self.assertTrue(
- explicit_datetime <= datetime.now(timezone.utc) - timedelta(days=5)
- and explicit_datetime >= datetime.now(timezone.utc) - timedelta(days=5, hours=1)
+ explicit_datetime <= datetime.now(UTC) - timedelta(days=5)
+ and explicit_datetime >= datetime.now(UTC) - timedelta(days=5, hours=1)
  )
 
  cohort_id = cohort["id"]

ee/session_recordings/session_summary/test/test_summarize_session.py

@@ -1,4 +1,4 @@
-from datetime import timezone, datetime
+from datetime import datetime, UTC
 
 from dateutil.parser import isoparse
 
@@ -23,7 +23,7 @@ def test_format_dates_as_millis_since_start(self) -> None:
  ["$pageview", isoparse("2021-01-01T00:00:02Z")],
  ],
  ),
- datetime(2021, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
+ datetime(2021, 1, 1, 0, 0, 0, tzinfo=UTC),
  )
  assert processed.columns == ["event", "milliseconds_since_start"]
  assert processed.results == [["$pageview", 0], ["$pageview", 1000], ["$pageview", 2000]]

ee/session_recordings/test/test_session_recording_extensions.py

@@ -1,5 +1,5 @@
 import gzip
-from datetime import timedelta, datetime, timezone
+from datetime import timedelta, datetime, UTC
 from secrets import token_urlsafe
 from unittest.mock import patch, MagicMock
 from uuid import uuid4
@@ -84,7 +84,7 @@ def test_can_build_different_object_storage_paths(self) -> None:
 
  def test_persists_recording_from_blob_ingested_storage(self):
  with self.settings(OBJECT_STORAGE_SESSION_RECORDING_BLOB_INGESTION_FOLDER=TEST_BUCKET):
- two_minutes_ago = (datetime.now() - timedelta(minutes=2)).replace(tzinfo=timezone.utc)
+ two_minutes_ago = (datetime.now() - timedelta(minutes=2)).replace(tzinfo=UTC)
 
  with freeze_time(two_minutes_ago):
  session_id = f"test_persists_recording_from_blob_ingested_storage-s1-{uuid4()}"

ee/session_recordings/test/test_session_recording_playlist.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, UTC
 from unittest import mock
 from unittest.mock import MagicMock, patch
 from uuid import uuid4
@@ -187,7 +187,7 @@ def test_get_pinned_recordings_for_playlist(self, mock_copy_objects: MagicMock)
 
  session_one = f"test_fetch_playlist_recordings-session1-{uuid4()}"
  session_two = f"test_fetch_playlist_recordings-session2-{uuid4()}"
- three_days_ago = (datetime.now() - timedelta(days=3)).replace(tzinfo=timezone.utc)
+ three_days_ago = (datetime.now() - timedelta(days=3)).replace(tzinfo=UTC)
 
  produce_replay_summary(
  team_id=self.team.id,
@@ -242,7 +242,7 @@ def test_fetch_playlist_recordings(self, mock_copy_objects: MagicMock, mock_list
 
  session_one = f"test_fetch_playlist_recordings-session1-{uuid4()}"
  session_two = f"test_fetch_playlist_recordings-session2-{uuid4()}"
- three_days_ago = (datetime.now() - timedelta(days=3)).replace(tzinfo=timezone.utc)
+ three_days_ago = (datetime.now() - timedelta(days=3)).replace(tzinfo=UTC)
 
  for session_id in [session_one, session_two]:
  produce_replay_summary(

ee/tasks/subscriptions/subscription_utils.py

@@ -56,7 +56,7 @@ def generate_assets(
  # Wait for all assets to be exported
  tasks = [exporter.export_asset.si(asset.id) for asset in assets]
  # run them one after the other, so we don't exhaust celery workers
- exports_expire = datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(
+ exports_expire = datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(
  minutes=settings.PARALLEL_ASSET_GENERATION_MAX_TIMEOUT_MINUTES
  )
  parallel_job = chain(*tasks).apply_async(expires=exports_expire, retry=False)

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.10
+python_version = 3.11
 plugins =
  mypy_django_plugin.main,
  mypy_drf_plugin.main,

posthog/api/app_metrics.py

@@ -90,9 +90,7 @@ def get_batch_export_runs_app_metrics_queryset(self, batch_export_id: str):
  after = self.request.GET.get("date_from", "-30d")
  before = self.request.GET.get("date_to", None)
  after_datetime = relative_date_parse(after, self.team.timezone_info)
- before_datetime = (
- relative_date_parse(before, self.team.timezone_info) if before else dt.datetime.now(dt.timezone.utc)
- )
+ before_datetime = relative_date_parse(before, self.team.timezone_info) if before else dt.datetime.now(dt.UTC)
  date_range = (after_datetime, before_datetime)
  runs = (
  BatchExportRun.objects.select_related("batch_export__destination")

posthog/api/authentication.py

@@ -290,7 +290,7 @@ def create(self, validated_data):
  user = None
 
  if user:
- user.requested_password_reset_at = datetime.datetime.now(datetime.timezone.utc)
+ user.requested_password_reset_at = datetime.datetime.now(datetime.UTC)
  user.save()
  token = password_reset_token_generator.make_token(user)
  send_password_reset(user.id, token)

posthog/api/comments.py

@@ -11,11 +11,13 @@
 
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import UserBasicSerializer
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.models.comment import Comment
 
 
 class CommentSerializer(serializers.ModelSerializer):
  created_by = UserBasicSerializer(read_only=True)
+ deleted = ClassicBehaviorBooleanFieldSerializer()
 
  class Meta:
  model = Comment

posthog/api/feature_flag.py

@@ -23,6 +23,7 @@
 from posthog.api.shared import UserBasicSerializer
 from posthog.api.tagged_item import TaggedItemSerializerMixin, TaggedItemViewSetMixin
 from posthog.api.dashboards.dashboard import Dashboard
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.auth import PersonalAPIKeyAuthentication, TemporaryTokenAuthentication
 from posthog.constants import FlagRequestType
 from posthog.event_usage import report_user_action
@@ -89,6 +90,9 @@ class FeatureFlagSerializer(TaggedItemSerializerMixin, serializers.HyperlinkedMo
  is_simple_flag = serializers.SerializerMethodField()
  rollout_percentage = serializers.SerializerMethodField()
 
+ ensure_experience_continuity = ClassicBehaviorBooleanFieldSerializer()
+ has_enriched_analytics = ClassicBehaviorBooleanFieldSerializer()
+
  experiment_set: serializers.PrimaryKeyRelatedField = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
  surveys: serializers.SerializerMethodField = serializers.SerializerMethodField()
  features: serializers.SerializerMethodField = serializers.SerializerMethodField()

posthog/api/plugin.py

@@ -22,6 +22,7 @@
 
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import FiltersSerializer
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.models import Plugin, PluginAttachment, PluginConfig, User
 from posthog.models.activity_logging.activity_log import (
  ActivityPage,
@@ -586,6 +587,8 @@ class PluginConfigSerializer(serializers.ModelSerializer):
  delivery_rate_24h = serializers.SerializerMethodField()
  error = serializers.SerializerMethodField()
 
+ deleted = ClassicBehaviorBooleanFieldSerializer()
+
  class Meta:
  model = PluginConfig
  fields = [

posthog/api/routing.py

@@ -36,6 +36,32 @@
 class DefaultRouterPlusPlus(ExtendedDefaultRouter):
  """DefaultRouter with optional trailing slash and drf-extensions nesting."""
 
+ # This is an override because of changes in djangorestframework 3.15, which is required for python 3.11
+ # changes taken from and explained here: https://github.com/nautobot/nautobot/pull/5546/files#diff-81850a2ccad5814aab4f477d447f85cc0a82e9c10fd88fd72327cda51a750471R30
+ def _register(self, prefix, viewset, basename=None):
+ """
+ Override DRF's BaseRouter.register() to bypass an unnecessary restriction added in version 3.15.0.
+ (Reference: https://github.com/encode/django-rest-framework/pull/8438)
+ """
+ if basename is None:
+ basename = self.get_default_basename(viewset)
+
+ # DRF:
+ # if self.is_already_registered(basename):
+ # msg = (f'Router with basename "{basename}" is already registered. '
+ # f'Please provide a unique basename for viewset "{viewset}"')
+ # raise ImproperlyConfigured(msg)
+ #
+ # We bypass this because we have at least one use case (/api/extras/jobs/) where we are *intentionally*
+ # registering two viewsets with the same basename, but have carefully defined them so as not to conflict.
+
+ # resuming standard DRF code...
+ self.registry.append((prefix, viewset, basename))
+
+ # invalidate the urls cache
+ if hasattr(self, "_urls"):
+ del self._urls
+
  def __init__(self, *args, **kwargs):
  super().__init__(*args, **kwargs)
  self.trailing_slash = r"/?"

posthog/api/test/__snapshots__/test_api_docs.ambr

@@ -77,7 +77,7 @@
  "/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Error [PropertyDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')",
  '/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Warning [PropertyDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.property_definition.PropertyDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".',
  '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. <int:project_id>) or annotating the parameter type with @extend_schema. Defaulting to "string".',
- '/opt/hostedtoolcache/Python/3.10.10/x64/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py: Warning [QueryViewSet > ModelMetaclass]: Encountered 2 components with identical names "Person" and different classes <class \'str\'> and <class \'posthog.api.person.PersonSerializer\'>. This will very likely result in an incorrect schema. Try renaming one.',
+ '/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py: Warning [QueryViewSet > ModelMetaclass]: Encountered 2 components with identical names "Person" and different classes <class \'str\'> and <class \'posthog.api.person.PersonSerializer\'>. This will very likely result in an incorrect schema. Try renaming one.',
  '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. <int:id>) or annotating the parameter type with @extend_schema. Defaulting to "string".',
  '/home/runner/work/posthog/posthog/posthog/api/query.py: Error [QueryViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.',
  '/home/runner/work/posthog/posthog/ee/session_recordings/session_recording_playlist.py: Warning [SessionRecordingPlaylistViewSet]: could not derive type of path parameter "project_id" because model "posthog.session_recordings.models.session_recording_playlist.SessionRecordingPlaylist" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".',