From 1ed89ffa6b55d448b768400857b9ce6c7eb2c3ea Mon Sep 17 00:00:00 2001 From: Brett Rowan <121075405+b-rowan@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:45:14 -0600 Subject: [PATCH 1/3] Remove aiofiles types from mypy dependencies Not needed anymore since aiofiles was removed. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d03a52c1..9589689a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ pre-commit = "^3.6.2" flake8 = "7.1.0" mypy = "^1.11.2" types-pyyaml = "^6.0.12.20240808" -types-aiofiles = "^24.1.0.20240626" [tool.poetry.group.docs.dependencies] mkdocs = "^1.6.0" From df3965b4828887a0f2146ace3862de26a7101705 Mon Sep 17 00:00:00 2001 From: Brett Rowan <121075405+b-rowan@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:48:30 -0600 Subject: [PATCH 2/3] Ensure software exists in DB before creating rollout When sending a rollout put request via the API, there was no validation that the referenced software file existed. --- goosebit/api/v1/rollouts/routes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/goosebit/api/v1/rollouts/routes.py b/goosebit/api/v1/rollouts/routes.py index 8c65a9da..4a4eb94c 100644 --- a/goosebit/api/v1/rollouts/routes.py +++ b/goosebit/api/v1/rollouts/routes.py @@ -1,9 +1,9 @@ -from fastapi import APIRouter, Security +from fastapi import APIRouter, HTTPException, Security from fastapi.requests import Request from goosebit.api.responses import StatusResponse from goosebit.auth import validate_user_permissions -from goosebit.db.models import Rollout +from goosebit.db.models import Rollout, Software from .requests import RolloutsDeleteRequest, RolloutsPatchRequest, RolloutsPutRequest from .responses import RolloutsPutResponse, RolloutsResponse @@ -24,6 +24,9 @@ async def rollouts_get(_: Request) -> RolloutsResponse: dependencies=[Security(validate_user_permissions, scopes=["rollout.write"])], ) async def rollouts_put(_: Request, rollout: RolloutsPutRequest) -> RolloutsPutResponse: + software = await Software.filter(id=rollout.software_id) + if len(software) == 0: + raise HTTPException(404, f"No software with ID {rollout.software_id} found") rollout = await Rollout.create( name=rollout.name, feed=rollout.feed, From 1fb63689e6462dc1674ed8dfb5b60ab4fe66412a Mon Sep 17 00:00:00 2001 From: Brett Rowan <121075405+b-rowan@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:49:37 -0600 Subject: [PATCH 3/3] Use `model_validate` built into pydantic to parse DB models `pydantic` models have a deprecated `from_orm` method, which is designed to parse DB models into the schema. This was superseded by `model_validate`, which does the same thing when the schema config `from_attributes` value is true. Migrating to using this simplifies the method of parsing DB models, and will eventually allow the schemas to be used internally instead of the DB models. --- goosebit/api/v1/devices/device/routes.py | 3 +- goosebit/api/v1/devices/responses.py | 7 --- goosebit/api/v1/devices/routes.py | 3 +- goosebit/api/v1/rollouts/responses.py | 9 +-- goosebit/api/v1/rollouts/routes.py | 3 +- goosebit/api/v1/software/responses.py | 7 --- goosebit/api/v1/software/routes.py | 5 +- goosebit/db/models.py | 2 + goosebit/schema/devices.py | 74 +++++++++++++----------- goosebit/schema/rollouts.py | 39 +++++++------ goosebit/schema/software.py | 43 ++++++++------ goosebit/ui/bff/devices/responses.py | 6 +- goosebit/ui/bff/rollouts/responses.py | 4 +- goosebit/ui/bff/rollouts/routes.py | 2 +- goosebit/ui/bff/software/responses.py | 4 +- 15 files changed, 103 insertions(+), 108 deletions(-) diff --git a/goosebit/api/v1/devices/device/routes.py b/goosebit/api/v1/devices/device/routes.py index 5d716d01..45f93697 100644 --- a/goosebit/api/v1/devices/device/routes.py +++ b/goosebit/api/v1/devices/device/routes.py @@ -18,7 +18,8 @@ async def device_get(_: Request, updater: UpdateManager = Depends(get_update_man device = await updater.get_device() if device is None: raise HTTPException(404) - return await DeviceResponse.convert(device) + await device.fetch_related("assigned_software", "hardware") + return DeviceResponse.model_validate(device) @router.get( diff --git a/goosebit/api/v1/devices/responses.py b/goosebit/api/v1/devices/responses.py index ae8472a5..ff73fc36 100644 --- a/goosebit/api/v1/devices/responses.py +++ b/goosebit/api/v1/devices/responses.py @@ -1,16 +1,9 @@ from __future__ import annotations -import asyncio - from pydantic import BaseModel -from goosebit.db.models import Device from goosebit.schema.devices import DeviceSchema class DevicesResponse(BaseModel): devices: list[DeviceSchema] - - @classmethod - async def convert(cls, devices: list[Device]): - return cls(devices=await asyncio.gather(*[DeviceSchema.convert(d) for d in devices])) diff --git a/goosebit/api/v1/devices/routes.py b/goosebit/api/v1/devices/routes.py index 073c0214..9f3af92b 100644 --- a/goosebit/api/v1/devices/routes.py +++ b/goosebit/api/v1/devices/routes.py @@ -20,7 +20,8 @@ dependencies=[Security(validate_user_permissions, scopes=["home.read"])], ) async def devices_get(_: Request) -> DevicesResponse: - return await DevicesResponse.convert(await Device.all().prefetch_related("assigned_software", "hardware")) + devices = await Device.all().prefetch_related("assigned_software", "hardware") + return DevicesResponse(devices=devices) @router.delete( diff --git a/goosebit/api/v1/rollouts/responses.py b/goosebit/api/v1/rollouts/responses.py index cc5373a8..e4c1b11a 100644 --- a/goosebit/api/v1/rollouts/responses.py +++ b/goosebit/api/v1/rollouts/responses.py @@ -1,19 +1,14 @@ -import asyncio +from __future__ import annotations from pydantic import BaseModel from goosebit.api.responses import StatusResponse -from goosebit.db.models import Rollout from goosebit.schema.rollouts import RolloutSchema class RolloutsPutResponse(StatusResponse): - id: int + id: int | None = None class RolloutsResponse(BaseModel): rollouts: list[RolloutSchema] - - @classmethod - async def convert(cls, devices: list[Rollout]): - return cls(rollouts=await asyncio.gather(*[RolloutSchema.convert(d) for d in devices])) diff --git a/goosebit/api/v1/rollouts/routes.py b/goosebit/api/v1/rollouts/routes.py index 4a4eb94c..6486a218 100644 --- a/goosebit/api/v1/rollouts/routes.py +++ b/goosebit/api/v1/rollouts/routes.py @@ -16,7 +16,8 @@ dependencies=[Security(validate_user_permissions, scopes=["rollout.read"])], ) async def rollouts_get(_: Request) -> RolloutsResponse: - return await RolloutsResponse.convert(await Rollout.all().prefetch_related("software")) + rollouts = await Rollout.all().prefetch_related("software", "software__compatibility") + return RolloutsResponse(rollouts=rollouts) @router.post( diff --git a/goosebit/api/v1/software/responses.py b/goosebit/api/v1/software/responses.py index 207eb259..c5808fd7 100644 --- a/goosebit/api/v1/software/responses.py +++ b/goosebit/api/v1/software/responses.py @@ -1,16 +1,9 @@ from __future__ import annotations -import asyncio - from pydantic import BaseModel -from goosebit.db.models import Software from goosebit.schema.software import SoftwareSchema class SoftwareResponse(BaseModel): software: list[SoftwareSchema] - - @classmethod - async def convert(cls, software: list[Software]): - return cls(software=await asyncio.gather(*[SoftwareSchema.convert(f) for f in software])) diff --git a/goosebit/api/v1/software/routes.py b/goosebit/api/v1/software/routes.py index 6f8da5d7..c15c2239 100644 --- a/goosebit/api/v1/software/routes.py +++ b/goosebit/api/v1/software/routes.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import random import string @@ -22,7 +24,8 @@ dependencies=[Security(validate_user_permissions, scopes=["software.read"])], ) async def software_get(_: Request) -> SoftwareResponse: - return await SoftwareResponse.convert(await Software.all().prefetch_related("compatibility")) + software = await Software.all().prefetch_related("compatibility") + return SoftwareResponse(software=software) @router.delete( diff --git a/goosebit/db/models.py b/goosebit/db/models.py index 979f800a..91e60ec0 100644 --- a/goosebit/db/models.py +++ b/goosebit/db/models.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from enum import IntEnum from typing import Self from urllib.parse import unquote, urlparse diff --git a/goosebit/schema/devices.py b/goosebit/schema/devices.py index bcb19733..ca48dff2 100644 --- a/goosebit/schema/devices.py +++ b/goosebit/schema/devices.py @@ -4,10 +4,11 @@ from enum import Enum, IntEnum, StrEnum from typing import Annotated -from pydantic import BaseModel, BeforeValidator, computed_field +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, computed_field -from goosebit.db.models import Device, UpdateModeEnum, UpdateStateEnum -from goosebit.updater.manager import get_update_manager +from goosebit.db.models import UpdateModeEnum, UpdateStateEnum +from goosebit.schema.software import HardwareSchema, SoftwareSchema +from goosebit.updater.manager import DeviceUpdateManager class ConvertableEnum(StrEnum): @@ -26,48 +27,51 @@ def enum_factory(name: str, base: type[Enum]) -> type[ConvertableEnum]: class DeviceSchema(BaseModel): + model_config = ConfigDict(from_attributes=True) + uuid: str name: str | None sw_version: str | None - sw_target_version: str | None - sw_assigned: int | None - hw_model: str - hw_revision: str + + assigned_software: SoftwareSchema | None = Field(exclude=True) + hardware: HardwareSchema | None = Field(exclude=True) + feed: str progress: int | None last_state: Annotated[UpdateStateSchema, BeforeValidator(UpdateStateSchema.convert)] # type: ignore[valid-type] update_mode: Annotated[UpdateModeSchema, BeforeValidator(UpdateModeSchema.convert)] # type: ignore[valid-type] force_update: bool last_ip: str | None - last_seen: int | None - poll_seconds: int + last_seen: Annotated[ + int | None, BeforeValidator(lambda last_seen: round(time.time() - last_seen) if last_seen is not None else None) + ] - @computed_field + @computed_field # type: ignore[misc] + @property def online(self) -> bool | None: return self.last_seen < self.poll_seconds if self.last_seen is not None else None - @classmethod - async def convert(cls, device: Device): - manager = await get_update_manager(device.uuid) - _, target_software = await manager.get_update() - last_seen = device.last_seen - if last_seen is not None: - last_seen = round(time.time() - device.last_seen) - - return cls( - uuid=device.uuid, - name=device.name, - sw_version=device.sw_version, - sw_target_version=(target_software.version if target_software is not None else None), - sw_assigned=(device.assigned_software.id if device.assigned_software is not None else None), - hw_model=device.hardware.model, - hw_revision=device.hardware.revision, - feed=device.feed, - progress=device.progress, - last_state=device.last_state, - update_mode=device.update_mode, - force_update=device.force_update, - last_ip=device.last_ip, - last_seen=last_seen, - poll_seconds=manager.poll_seconds, - ) + @computed_field # type: ignore[misc] + @property + def sw_target_version(self) -> str | None: + return self.assigned_software.version if self.assigned_software is not None else None + + @computed_field # type: ignore[misc] + @property + def sw_assigned(self) -> int | None: + return self.assigned_software.id if self.assigned_software is not None else None + + @computed_field # type: ignore[misc] + @property + def hw_model(self) -> str | None: + return self.hardware.model if self.hardware is not None else None + + @computed_field # type: ignore[misc] + @property + def hw_revision(self) -> str | None: + return self.hardware.revision if self.hardware is not None else None + + @computed_field # type: ignore[misc] + @property + def poll_seconds(self) -> int: + return DeviceUpdateManager(self.uuid).poll_seconds diff --git a/goosebit/schema/rollouts.py b/goosebit/schema/rollouts.py index 53452467..89f29396 100644 --- a/goosebit/schema/rollouts.py +++ b/goosebit/schema/rollouts.py @@ -1,31 +1,34 @@ from __future__ import annotations -from pydantic import BaseModel +from datetime import datetime -from goosebit.db.models import Rollout +from pydantic import BaseModel, ConfigDict, Field, computed_field, field_serializer + +from goosebit.schema.software import SoftwareSchema class RolloutSchema(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: int - created_at: int + created_at: datetime name: str | None feed: str - sw_file: str - sw_version: str + software: SoftwareSchema = Field(exclude=True) paused: bool success_count: int failure_count: int - @classmethod - async def convert(cls, rollout: Rollout): - return cls( - id=rollout.id, - created_at=int(rollout.created_at.timestamp() * 1000), - name=rollout.name, - feed=rollout.feed, - sw_file=rollout.software.path.name, - sw_version=rollout.software.version, - paused=rollout.paused, - success_count=rollout.success_count, - failure_count=rollout.failure_count, - ) + @computed_field # type: ignore[misc] + @property + def sw_version(self) -> str: + return self.software.version + + @computed_field # type: ignore[misc] + @property + def sw_file(self) -> str: + return self.software.path.name + + @field_serializer("created_at") + def serialize_created_at(self, created_at: datetime, _info): + return int(created_at.timestamp() * 1000) diff --git a/goosebit/schema/software.py b/goosebit/schema/software.py index b648c5de..7b9bb228 100644 --- a/goosebit/schema/software.py +++ b/goosebit/schema/software.py @@ -1,37 +1,42 @@ from __future__ import annotations -import asyncio +from urllib.parse import unquote, urlparse +from urllib.request import url2pathname -from pydantic import BaseModel - -from goosebit.db.models import Hardware, Software +from anyio import Path +from pydantic import BaseModel, ConfigDict, Field, computed_field class HardwareSchema(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: int model: str revision: str - @classmethod - async def convert(cls, hardware: Hardware): - return cls(id=hardware.id, model=hardware.model, revision=hardware.revision) - class SoftwareSchema(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: int - name: str + uri: str = Field(exclude=True) size: int hash: str version: str compatibility: list[HardwareSchema] - @classmethod - async def convert(cls, software: Software): - return cls( - id=software.id, - name=software.path_user, - size=software.size, - hash=software.hash, - version=software.version, - compatibility=await asyncio.gather(*[HardwareSchema.convert(h) for h in software.compatibility]), - ) + @property + def path(self) -> Path: + return Path(url2pathname(unquote(urlparse(self.uri).path))) + + @property + def local(self) -> bool: + return urlparse(self.uri).scheme == "file" + + @computed_field # type: ignore[misc] + @property + def name(self) -> str: + if self.local: + return self.path.name + else: + return self.uri diff --git a/goosebit/ui/bff/devices/responses.py b/goosebit/ui/bff/devices/responses.py index d60f36c3..728945ba 100644 --- a/goosebit/ui/bff/devices/responses.py +++ b/goosebit/ui/bff/devices/responses.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio - from fastapi.requests import Request from pydantic import BaseModel, Field @@ -33,7 +31,7 @@ async def convert(cls, request: Request, query, search_filter, total_records): query = query.order_by(f"{'-' if order_dir == 'desc' else ''}{order_column}") filtered_records = await query.count() - devices = await query.offset(start).limit(length).all() - data = await asyncio.gather(*[DeviceSchema.convert(d) for d in devices]) + devices = await query.offset(start).limit(length).all().prefetch_related("assigned_software", "hardware") + data = [DeviceSchema.model_validate(d) for d in devices] return cls(data=data, draw=draw, records_total=total_records, records_filtered=filtered_records) diff --git a/goosebit/ui/bff/rollouts/responses.py b/goosebit/ui/bff/rollouts/responses.py index f47c3bba..b9862db6 100644 --- a/goosebit/ui/bff/rollouts/responses.py +++ b/goosebit/ui/bff/rollouts/responses.py @@ -1,5 +1,3 @@ -import asyncio - from fastapi.requests import Request from pydantic import BaseModel, Field @@ -32,6 +30,6 @@ async def convert(cls, request: Request, query, search_filter, total_records): filtered_records = await query.count() rollouts = await query.offset(start).limit(length).all() - data = await asyncio.gather(*[RolloutSchema.convert(r) for r in rollouts]) + data = [RolloutSchema.model_validate(r) for r in rollouts] return cls(data=data, draw=draw, records_total=total_records, records_filtered=filtered_records) diff --git a/goosebit/ui/bff/rollouts/routes.py b/goosebit/ui/bff/rollouts/routes.py index beb95f91..aa55475a 100644 --- a/goosebit/ui/bff/rollouts/routes.py +++ b/goosebit/ui/bff/rollouts/routes.py @@ -19,7 +19,7 @@ async def rollouts_get(request: Request) -> BFFRolloutsResponse: def search_filter(search_value): return Q(name__icontains=search_value) | Q(feed__icontains=search_value) - query = Rollout.all().prefetch_related("software") + query = Rollout.all().prefetch_related("software", "software__compatibility") total_records = await Rollout.all().count() return await BFFRolloutsResponse.convert(request, query, search_filter, total_records) diff --git a/goosebit/ui/bff/software/responses.py b/goosebit/ui/bff/software/responses.py index 587b7f84..d01f48c0 100644 --- a/goosebit/ui/bff/software/responses.py +++ b/goosebit/ui/bff/software/responses.py @@ -1,5 +1,3 @@ -import asyncio - from fastapi.requests import Request from pydantic import BaseModel, Field @@ -32,6 +30,6 @@ async def convert(cls, request: Request, query, search_filter, total_records): filtered_records = await query.count() devices = await query.offset(start).limit(length).all() - data = await asyncio.gather(*[SoftwareSchema.convert(d) for d in devices]) + data = [SoftwareSchema.model_validate(d) for d in devices] return cls(data=data, draw=draw, records_total=total_records, records_filtered=filtered_records)