Multiple changes:

- Upgrade all images to python3.11
- Upgrade all requirements
- Upgrade CI configs
- Remove everything kubernetes-related due to support difficulty

Author: pomo-mondreganto

.github/workflows/release.yml

@@ -10,28 +10,33 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Get the version
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: amd64,arm64
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          platforms: linux/amd64,linux/arm64
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ github.token }}
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker_config/base_images/backend.Dockerfile
@@ -42,7 +47,7 @@ jobs:
             ghcr.io/${{ github.repository_owner }}/forcad_base:${{ steps.get_version.outputs.VERSION }}
 
       - name: Set up Node 16
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: "16"
 

.github/workflows/tests.yml

@@ -30,29 +30,26 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - name: Setup python
-        uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements.txt') }}-${{ hashFiles('cli/requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Setup tests
-        run: python tests/setup_forcad.py
+          python-version: "3.11"
+          cache: "pip"
+          cache-dependency-path: |
+            cli/requirements.txt
+            tests/requirements.txt
+            backend/requirements.txt
 
       - name: Install requirements
         run: |
           pip install -r cli/requirements.txt
           pip install -r tests/requirements.txt
           pip install -r backend/requirements.txt
 
+      - name: Setup tests
+        run: python tests/setup_forcad.py
+
       - name: Setup ForcAD
         run: ./control.py setup
 
@@ -79,12 +76,11 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - name: Setup python
-        uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.11"
 
       - name: Install dependencies
         run: pip install flake8
@@ -99,10 +95,10 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Node 16
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: "16"
 
@@ -123,25 +119,3 @@ jobs:
       - name: Check the frontend compiles
         working-directory: front
         run: yarn build
-
-  terraform_checks:
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: deploy/terraform
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v1
-
-      - name: Terraform fmt
-        run: terraform fmt -check
-        continue-on-error: true
-
-      - name: Terraform Init
-        run: terraform init
-
-      - name: Terraform Validate
-        run: terraform validate

backend/lib/config/models.py

@@ -2,34 +2,37 @@
 from typing import List
 
 from pydantic import BaseModel, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
-def env_field(key: str) -> Field:
-    return Field(default_factory=lambda: os.environ[key])
+class Redis(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='redis_')
 
-
-class Redis(BaseModel):
-    host: str = env_field('REDIS_HOST')
-    port: int = env_field('REDIS_PORT')
-    password: str = env_field('REDIS_PASSWORD')
+    host: str
+    port: int
+    password: str
     db: int = 0
 
     @property
     def url(self) -> str:
         return f'redis://:{self.password}@{self.host}:{self.port}/{self.db}'
 
 
-class WebCredentials(BaseModel):
-    username: str = env_field('ADMIN_USERNAME')
-    password: str = env_field('ADMIN_PASSWORD')
+class WebCredentials(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='admin_')
+
+    username: str
+    password: str
+
 
+class Database(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='postgres_')
 
-class Database(BaseModel):
-    host: str = env_field('POSTGRES_HOST')
-    port: int = env_field('POSTGRES_PORT')
-    user: str = env_field('POSTGRES_USER')
-    password: str = env_field('POSTGRES_PASSWORD')
-    dbname: str = env_field('POSTGRES_DB')
+    host: str
+    port: int
+    user: str
+    password: str
+    dbname: str = Field(validation_alias='postgres_db')
 
 
 class Celery(BaseModel):
@@ -39,7 +42,7 @@ class Celery(BaseModel):
 
     worker_prefetch_multiplier: int = 1
 
-    result_expires = 15 * 60
+    result_expires: int = 15 * 60
     redis_socket_timeout: int = 10
     redis_socket_keepalive: bool = True
     redis_retry_on_timeout: bool = True

backend/lib/storage/utils.py

@@ -1,12 +1,12 @@
 from contextlib import contextmanager
+from typing import TypeVar
 
 import kombu
 import redis
 import socketio
-from psycopg2 import pool, extras
-
 from lib import config
-from lib.helpers.singleton import Singleton, T
+from lib.helpers.singleton import Singleton
+from psycopg2 import extras, pool
 
 
 class DBPool(Singleton[pool.SimpleConnectionPool]):
@@ -17,7 +17,7 @@ def create() -> pool.SimpleConnectionPool:
         return pool.SimpleConnectionPool(
             minconn=5,
             maxconn=20,
-            **database_config.dict(),
+            **database_config.model_dump(),
         )
 
 
@@ -26,7 +26,7 @@ class RedisStorage(Singleton[redis.Redis]):
     @staticmethod
     def create() -> redis.Redis:
         redis_config = config.get_redis_config()
-        return redis.Redis(decode_responses=True, **redis_config.dict())
+        return redis.Redis(decode_responses=True, **redis_config.model_dump())
 
 
 class SIOManager(Singleton[socketio.KombuManager]):
@@ -41,7 +41,7 @@ def create(*, write_only: bool = False) -> socketio.KombuManager:
         )
 
     @classmethod
-    def get(cls, *, write_only: bool) -> T:
+    def get(cls, *, write_only: bool) -> socketio.KombuManager:
         return super().get(write_only=write_only)
 
     @classmethod

backend/requirements.txt

@@ -1,54 +1,54 @@
-amqp==5.0.9
-async-timeout==4.0.2
-autopep8==1.6.0
-bidict==0.21.2
-billiard==3.6.4.0
-celery==5.2.7
-certifi==2020.12.5
-charset-normalizer==2.1.0
-click==8.0.3
-click-didyoumean==0.0.3
+amqp==5.1.1
+annotated-types==0.6.0
+bidict==0.22.1
+billiard==4.1.0
+blinker==1.6.3
+celery==5.3.4
+certifi==2023.7.22
+charset-normalizer==3.3.1
+click==8.1.7
+click-didyoumean==0.3.0
 click-plugins==1.1.1
-click-repl==0.2.0
-Deprecated==1.2.13
-dnspython==1.16.0
-eventlet==0.30.2
-Flask==2.2.2
-Flask-Cors==3.0.10
-Flask-SocketIO==5.3.2
-flower==1.2.0
-greenlet==1.1.2
-gunicorn==20.1.0
-hiredis==2.0.0
-humanize==3.12.0
-idna==3.3
-importlib-metadata==4.12.0
-itsdangerous==2.0.1
+click-repl==0.3.0
+dnspython==2.4.2
+eventlet==0.33.3
+Flask==3.0.0
+Flask-Cors==4.0.0
+Flask-SocketIO==5.3.6
+flower==2.0.1
+greenlet==3.0.0
+gunicorn==21.2.0
+h11==0.14.0
+hiredis==2.2.3
+humanize==4.8.0
+idna==3.4
+itsdangerous==2.1.2
 Jinja2==3.1.2
-kombu==5.2.3
-MarkupSafe==2.1.1
-packaging==21.3
-prometheus-client==0.15.0
-prometheus-flask-exporter==0.21.0
-prompt-toolkit==3.0.18
-psycopg2==2.9.5
-pycodestyle==2.8.0
-pydantic==1.10.2
-pyparsing==3.0.6
-python-dateutil==2.8.1
-python-engineio==4.3.3
-python-socketio==5.7.2
-pytz==2022.6
-PyYAML==6.0
-redis==4.3.5
-requests==2.28.1
+kombu==5.3.2
+MarkupSafe==2.1.3
+packaging==23.2
+prometheus-client==0.17.1
+prometheus-flask-exporter==0.22.4
+prompt-toolkit==3.0.39
+psycopg2==2.9.9
+pydantic==2.4.2
+pydantic-settings==2.0.3
+pydantic_core==2.10.1
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-engineio==4.8.0
+python-socketio==5.10.0
+pytz==2023.3.post1
+PyYAML==6.0.1
+redis==5.0.1
+requests==2.31.0
+simple-websocket==1.0.0
 six==1.16.0
-toml==0.10.2
-tornado==6.1
-typing_extensions==4.4.0
-urllib3==1.26.10
+tornado==6.3.3
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.0.7
 vine==5.0.0
-wcwidth==0.2.5
-Werkzeug==2.2.2
-wrapt==1.13.3
-zipp==3.8.1
+wcwidth==0.2.8
+Werkzeug==3.0.0
+wsproto==1.2.0

backend/services/admin/viewsets/api_base.py

@@ -7,7 +7,7 @@
 
 
 class ApiSet:
-    model: str = None
+    model: str
 
     def get_id_kwarg(self) -> str:
         return f'{self.model}_id'

backend/services/tasks/celery_factory.py

@@ -1,5 +1,4 @@
 from celery import Celery
-
 from lib import config
 
 
@@ -13,5 +12,5 @@ def get_celery_app():
         ],
     )
 
-    app.conf.update(celery_config.dict())
+    app.conf.update(celery_config.model_dump())
     return app

backend/services/tasks/handlers.py

@@ -3,16 +3,16 @@
 from celery import shared_task
 from celery.result import AsyncResult
 from celery.utils.log import get_task_logger
-
-from lib import storage, models
+from lib import models, storage
 from lib.helpers.jobs import JobNames
-from lib.models import TaskStatus, Action
+from lib.models import Action, TaskStatus
 
 logger = get_task_logger(__name__)
 
 
 @shared_task(name=JobNames.error_handler)
 def exception_callback(result: AsyncResult, exc: Exception, traceback: str) -> None:
+    print('!!!', result, type(result))
     action_name = result.task.split('.')[-1].split('_')[0].upper()
     action = Action[action_name]