chore: upgrade python to 3.11 🐍 (#22932)

🐍

.devcontainer/Dockerfile

@@ -1,3 +1,7 @@
+# hadolint global ignore=DL3004
+
+# hadolint doesn't like changes to this file, but it is only used for local dev
+
 # Defines the environment you're dropped into with codespaces
 # I've take
 # https://github.com/microsoft/vscode-dev-containers/blob/main/containers/python-3/.devcontainer/Dockerfile
@@ -7,7 +11,7 @@
 # experience as rich as possible. Perhaps later down the line it might be worth
 # rolling our own
 #
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bullseye
 
 # Make sure all exit codes on pipes cause failures
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

.github/actions/run-backend-tests/action.yml

@@ -6,7 +6,7 @@ name: Run Django tests
 inputs:
  python-version:
  required: true
- description: Python version, e.g. 3.10.10
+ description: Python version, e.g. 3.11.9
  clickhouse-server-image:
  required: true
  description: ClickHouse server image tag, e.g. clickhouse/clickhouse-server:latest

.github/workflows/benchmark.yml

@@ -54,7 +54,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-backend-update-test-timing.yml

@@ -28,7 +28,7 @@ jobs:
  concurrency: 1
  group: 1
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- python-version: '3.10.10'
+ python-version: '3.11.9'
  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
  segment: 'FOSS'
  person-on-events: false

.github/workflows/ci-backend.yml

@@ -108,7 +108,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -163,7 +163,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -177,22 +177,21 @@ jobs:
  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
  # First running migrations from master, to simulate the real-world scenario
-
- - name: Checkout master
- uses: actions/checkout@v3
- with:
- ref: master
-
- - name: Install python dependencies for master
- run: |
- uv pip install --system -r requirements.txt -r requirements-dev.txt
-
- - name: Run migrations up to master
- run: |
- python manage.py migrate
+ # Commented out to move to Python 3.11. Uncomment after deploy.
+ # - name: Checkout master
+ #  uses: actions/checkout@v3
+ #  with:
+ #  ref: master
+ #
+ # - name: Install python dependencies for master
+ #  run: |
+ #  uv pip install --system -r requirements.txt -r requirements-dev.txt
+ #
+ # - name: Run migrations up to master
+ #  run: |
+ #  python manage.py migrate
 
  # Now we can consider this PR's migrations
-
  - name: Checkout this PR
  uses: actions/checkout@v3
 
@@ -204,22 +203,24 @@ jobs:
  run: |
  python manage.py migrate
 
- - name: Check migrations
- run: |
- python manage.py makemigrations --check --dry-run
- git fetch origin master
- # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
- # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
- # In this check we exclusively care about files that were
- # added (A) in posthog/migrations/. We also want to ignore
- # initial migrations (0001_*) as these are guaranteed to be
- # run on initial setup where there is no data.
- git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe
-
- - name: Check CH migrations
- run: |
- # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
- git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' | python manage.py test_ch_migrations_are_safe
+ # Commented out to move to Python 3.11. Uncomment after deploy.
+ #
+ # - name: Check migrations
+ # run: |
+ # python manage.py makemigrations --check --dry-run
+ # git fetch origin master
+ # # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
+ # # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
+ # # In this check we exclusively care about files that were
+ # # added (A) in posthog/migrations/. We also want to ignore
+ # # initial migrations (0001_*) as these are guaranteed to be
+ # # run on initial setup where there is no data.
+ # git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe
+ #
+ # - name: Check CH migrations
+ # run: |
+ # # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
+ # git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' | python manage.py test_ch_migrations_are_safe
 
  django:
  needs: changes
@@ -231,7 +232,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ['3.10.10']
+ python-version: ['3.11.9']
  clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
  segment: ['Core']
  person-on-events: [false, true]
@@ -242,7 +243,7 @@ jobs:
  - segment: 'Temporal'
  person-on-events: false
  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
- python-version: '3.10.10'
+ python-version: '3.11.9'
  concurrency: 1
  group: 1
 
@@ -330,7 +331,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-hog.yml

@@ -70,7 +70,7 @@ jobs:
  if: needs.changes.outputs.hog == 'true'
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-plugin-server.yml

@@ -115,7 +115,7 @@ jobs:
  if: needs.changes.outputs.plugin-server == 'true'
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -207,7 +207,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: 3.10.10
+ python-version: 3.11.9
  cache: 'pip'
  cache-dependency-path: '**/requirements*.txt'
  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.run/Celery Beat.run.xml

@@ -14,6 +14,7 @@
  <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
  </envs>
  <option name="SDK_HOME" value="" />
+ <option name="SDK_NAME" value="Python 3.11 (posthog)" />
  <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
  <option name="IS_MODULE_SDK" value="true" />
  <option name="ADD_CONTENT_ROOTS" value="true" />

.run/Celery Threads.run.xml

@@ -14,6 +14,7 @@
  <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
  </envs>
  <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
+ <option name="SDK_NAME" value="Python 3.11 (posthog)" />
  <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
  <option name="IS_MODULE_SDK" value="true" />
  <option name="ADD_CONTENT_ROOTS" value="true" />
@@ -28,4 +29,4 @@
  <option name="INPUT_FILE" value="" />
  <method v="2" />
  </configuration>
-</component>
+</component>

.run/Celery.run.xml

@@ -5,17 +5,18 @@
  <option name="INTERPRETER_OPTIONS" value="" />
  <option name="PARENT_ENVS" value="true" />
  <envs>
+ <env name="PYTHONUNBUFFERED" value="1" />
  <env name="CLICKHOUSE_SECURE" value="False" />
  <env name="DATABASE_URL" value="postgres://posthog:posthog@localhost:5432/posthog" />
  <env name="DEBUG" value="1" />
  <env name="KAFKA_HOSTS" value="localhost" />
- <env name="PYTHONUNBUFFERED" value="1" />
  <env name="SKIP_SERVICE_VERSION_REQUIREMENTS" value="1" />
  <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
  </envs>
  <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
+ <option name="SDK_NAME" value="Python 3.11 (posthog)" />
  <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
- <option name="IS_MODULE_SDK" value="false" />
+ <option name="IS_MODULE_SDK" value="true" />
  <option name="ADD_CONTENT_ROOTS" value="true" />
  <option name="ADD_SOURCE_ROOTS" value="true" />
  <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />

.run/PostHog.run.xml

@@ -1,9 +1,11 @@
 <component name="ProjectRunConfigurationManager">
  <configuration default="false" name="PostHog" type="Python.DjangoServer" factoryName="Django server">
  <module name="posthog" />
+ <option name="ENV_FILES" value="" />
  <option name="INTERPRETER_OPTIONS" value="" />
  <option name="PARENT_ENVS" value="true" />
  <envs>
+ <env name="PYTHONUNBUFFERED" value="1" />
  <env name="BILLING_SERVICE_URL" value="https://billing.dev.posthog.dev" />
  <env name="CAPTURE_TIME_TO_SEE_DATA" value="0" />
  <env name="CLICKHOUSE_SECURE" value="False" />
@@ -14,14 +16,13 @@
  <env name="KEA_VERBOSE_LOGGING" value="false" />
  <env name="PRINT_SQL" value="1" />
  <env name="PYDEVD_USE_CYTHON" value="NO" />
- <env name="PYTHONUNBUFFERED" value="1" />
  <env name="SESSION_RECORDING_KAFKA_COMPRESSION" value="gzip" />
  <env name="SESSION_RECORDING_KAFKA_HOSTS" value="localhost" />
  <env name="SESSION_RECORDING_KAFKA_MAX_REQUEST_SIZE_BYTES" value="20971520" />
  <env name="SKIP_SERVICE_VERSION_REQUIREMENTS" value="1" />
  </envs>
  <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
- <option name="SDK_NAME" value="Python 3.10 (posthog)" />
+ <option name="SDK_NAME" value="Python 3.11 (posthog)" />
  <option name="WORKING_DIRECTORY" value="" />
  <option name="IS_MODULE_SDK" value="false" />
  <option name="ADD_CONTENT_ROOTS" value="true" />
@@ -47,4 +48,4 @@
  <option name="customRunCommand" value="" />
  <method v="2" />
  </configuration>
-</component>
+</component>

bin/build-schema-python.sh

@@ -4,7 +4,7 @@ set -e
 
 # Generate schema.py from schema.json
 datamodel-codegen \
- --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp \
+ --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.11 --disable-timestamp \
  --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum \
  --input frontend/src/queries/schema.json --input-file-type jsonschema \
  --output posthog/schema.py --output-model-type pydantic_v2.BaseModel \
@@ -29,3 +29,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
 else
  sed -i -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
 fi
+
+# Replace class Foo(str, Enum) with class Foo(StrEnum) for proper handling in format strings in python 3.11
+# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1313 is resolved
+
+sed -i -e 's/str, Enum/StrEnum/g' posthog/schema.py
+sed -i 's/from enum import Enum/from enum import Enum, StrEnum/g' posthog/schema.py

bin/deploy-hobby

@@ -184,7 +184,7 @@ if ! command -v docker &> /dev/null; then
  # Setup Docker
  sudo apt install -y apt-transport-https ca-certificates curl software-properties-common
  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add -
- sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
+ sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable"
  sudo apt update
  sudo apt-cache policy docker-ce
  sudo apt install -y docker-ce

ee/api/test/__snapshots__/test_time_to_see_data.ambr

@@ -20,7 +20,7 @@
  "first_name": "",
  "last_name": "",
  "email": "",
- "is_email_verified": false
+ "is_email_verified": null
  }
  },
  "children": [

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.10
+python_version = 3.11
 plugins =
  mypy_django_plugin.main,
  mypy_drf_plugin.main,

posthog/api/comments.py

@@ -11,11 +11,13 @@
 
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import UserBasicSerializer
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.models.comment import Comment
 
 
 class CommentSerializer(serializers.ModelSerializer):
  created_by = UserBasicSerializer(read_only=True)
+ deleted = ClassicBehaviorBooleanFieldSerializer()
 
  class Meta:
  model = Comment

posthog/api/feature_flag.py

@@ -23,6 +23,7 @@
 from posthog.api.shared import UserBasicSerializer
 from posthog.api.tagged_item import TaggedItemSerializerMixin, TaggedItemViewSetMixin
 from posthog.api.dashboards.dashboard import Dashboard
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.auth import PersonalAPIKeyAuthentication, TemporaryTokenAuthentication
 from posthog.constants import FlagRequestType
 from posthog.event_usage import report_user_action
@@ -89,6 +90,9 @@ class FeatureFlagSerializer(TaggedItemSerializerMixin, serializers.HyperlinkedMo
  is_simple_flag = serializers.SerializerMethodField()
  rollout_percentage = serializers.SerializerMethodField()
 
+ ensure_experience_continuity = ClassicBehaviorBooleanFieldSerializer()
+ has_enriched_analytics = ClassicBehaviorBooleanFieldSerializer()
+
  experiment_set: serializers.PrimaryKeyRelatedField = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
  surveys: serializers.SerializerMethodField = serializers.SerializerMethodField()
  features: serializers.SerializerMethodField = serializers.SerializerMethodField()

posthog/api/plugin.py

@@ -22,6 +22,7 @@
 
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import FiltersSerializer
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.models import Plugin, PluginAttachment, PluginConfig, User
 from posthog.models.activity_logging.activity_log import (
  ActivityPage,
@@ -586,6 +587,8 @@ class PluginConfigSerializer(serializers.ModelSerializer):
  delivery_rate_24h = serializers.SerializerMethodField()
  error = serializers.SerializerMethodField()
 
+ deleted = ClassicBehaviorBooleanFieldSerializer()
+
  class Meta:
  model = PluginConfig
  fields = [

posthog/api/routing.py

@@ -36,6 +36,32 @@
 class DefaultRouterPlusPlus(ExtendedDefaultRouter):
  """DefaultRouter with optional trailing slash and drf-extensions nesting."""
 
+ # This is an override because of changes in djangorestframework 3.15, which is required for python 3.11
+ # changes taken from and explained here: https://github.com/nautobot/nautobot/pull/5546/files#diff-81850a2ccad5814aab4f477d447f85cc0a82e9c10fd88fd72327cda51a750471R30
+ def _register(self, prefix, viewset, basename=None):
+ """
+ Override DRF's BaseRouter.register() to bypass an unnecessary restriction added in version 3.15.0.
+ (Reference: https://github.com/encode/django-rest-framework/pull/8438)
+ """
+ if basename is None:
+ basename = self.get_default_basename(viewset)
+
+ # DRF:
+ # if self.is_already_registered(basename):
+ # msg = (f'Router with basename "{basename}" is already registered. '
+ # f'Please provide a unique basename for viewset "{viewset}"')
+ # raise ImproperlyConfigured(msg)
+ #
+ # We bypass this because we have at least one use case (/api/extras/jobs/) where we are *intentionally*
+ # registering two viewsets with the same basename, but have carefully defined them so as not to conflict.
+
+ # resuming standard DRF code...
+ self.registry.append((prefix, viewset, basename))
+
+ # invalidate the urls cache
+ if hasattr(self, "_urls"):
+ del self._urls
+
  def __init__(self, *args, **kwargs):
  super().__init__(*args, **kwargs)
  self.trailing_slash = r"/?"