From fc90e3ea369ec4e8818d864030e4105063cc5316 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 7 Sep 2023 17:47:40 +0100 Subject: [PATCH] Added Table Schema (#1) * Bootstrapped table schema * Bootstrapped reference generator * Bootstrapped tests * Improved schema model * Added validate assignment --- docs/reference/schema.md | 3 + dpspecs/__init__.py | 3 + dpspecs/models/__init__.py | 3 + dpspecs/models/base.py | 25 +++++++ dpspecs/models/schema.py | 137 ++++++++++++++++++++++++++++++++++++ mkdocs.yaml | 11 ++- pyproject.toml | 1 + tests/models/__init__.py | 0 tests/models/test_schema.py | 6 ++ 9 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 docs/reference/schema.md create mode 100644 dpspecs/models/__init__.py create mode 100644 dpspecs/models/base.py create mode 100644 dpspecs/models/schema.py create mode 100644 tests/models/__init__.py create mode 100644 tests/models/test_schema.py diff --git a/docs/reference/schema.md b/docs/reference/schema.md new file mode 100644 index 0000000..c5db722 --- /dev/null +++ b/docs/reference/schema.md @@ -0,0 +1,3 @@ +# Schema + +::: dpspecs.Schema diff --git a/dpspecs/__init__.py b/dpspecs/__init__.py index e69de29..46a74fd 100644 --- a/dpspecs/__init__.py +++ b/dpspecs/__init__.py @@ -0,0 +1,3 @@ +from .models import * + +__all__ = ["Schema"] diff --git a/dpspecs/models/__init__.py b/dpspecs/models/__init__.py new file mode 100644 index 0000000..c1c4c62 --- /dev/null +++ b/dpspecs/models/__init__.py @@ -0,0 +1,3 @@ +from .schema import Schema + +__all__ = ["Schema"] diff --git a/dpspecs/models/base.py b/dpspecs/models/base.py new file mode 100644 index 0000000..1e45d29 --- /dev/null +++ b/dpspecs/models/base.py @@ -0,0 +1,25 @@ +from typing import Any, Dict, List + +from pydantic import BaseModel, ValidationError +from pydantic_core import ErrorDetails + + +class Model(BaseModel, validate_assignment=True): + def __str__(self): + return str(self.to_descriptor()) + + @classmethod + def validate_descriptor(cls, descriptor: Dict[str, Any]): + errors: List[ErrorDetails] = [] + try: + cls.model_validate(descriptor) + except ValidationError as e: + errors = e.errors() + return errors + + @classmethod + def from_descriptor(cls, descriptor: Dict[str, Any]): + return cls(**descriptor) + + def to_descriptor(self): + return self.model_dump(exclude_unset=True, exclude_none=True) diff --git a/dpspecs/models/schema.py b/dpspecs/models/schema.py new file mode 100644 index 0000000..25bed77 --- /dev/null +++ b/dpspecs/models/schema.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Literal, Optional, Union + +import pydantic +from typing_extensions import Annotated + +from .base import Model + +# Schema + + +class Schema(Model): + """Schema model""" + + fields: List[Field] + """List of fields""" + + missingValues: Optional[List[str]] = None + primaryKey: Optional[List[str]] = None + foreignKeys: Optional[List[ForeignKey]] = None + + +# Fields + + +class BaseField(Model): + name: str + type: str + title: Optional[str] = None + description: Optional[str] = None + format: Optional[str] = None + missingValues: Optional[List[str]] = None + + +class AnyField(BaseField): + type: Literal["any"] = "any" + + +class ArrayField(BaseField): + type: Literal["array"] = "array" + # support json/csv format + arrayItem: Optional[Dict[str, Any]] = None + + +class BooleanField(BaseField): + type: Literal["boolean"] = "boolean" + trueValues: Optional[List[str]] = None + falseValues: Optional[List[str]] = None + + +class DateField(BaseField): + type: Literal["date"] = "date" + + +class DatetimeField(BaseField): + type: Literal["datetime"] = "datetime" + + +class DurationField(BaseField): + type: Literal["duration"] = "duration" + + +class GeojsonField(BaseField): + type: Literal["geojson"] = "geojson" + + +class GeopointField(BaseField): + type: Literal["geopoint"] = "geopoint" + + +class IntegerField(BaseField): + type: Literal["integer"] = "integer" + bareNumber: Optional[bool] = None + groupChar: Optional[str] = None + + +class NumberField(BaseField): + type: Literal["number"] = "number" + bareNumber: Optional[bool] = None + groupChar: Optional[str] = None + decimalChar: Optional[str] = None + + +class ObjectField(BaseField): + type: Literal["object"] = "object" + + +class StringField(BaseField): + type: Literal["string"] = "string" + + +class TimeField(BaseField): + type: Literal["time"] = "time" + + +class YearField(BaseField): + type: Literal["year"] = "year" + + +class YearmonthField(BaseField): + type: Literal["yearmonth"] = "yearmonth" + + +Field = Annotated[ + Union[ + AnyField, + ArrayField, + BooleanField, + DateField, + DatetimeField, + DurationField, + GeojsonField, + GeopointField, + IntegerField, + NumberField, + ObjectField, + StringField, + TimeField, + YearField, + YearmonthField, + ], + pydantic.Field(discriminator="type"), +] + + +# Foreign keys + + +class ForeignKey(Model): + fields: List[str] + reference: Optional[ForeignKeyReference] = None + + +class ForeignKeyReference(Model): + fields: List[str] + resource: str diff --git a/mkdocs.yaml b/mkdocs.yaml index 6cba16b..b705d22 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -28,7 +28,7 @@ theme: # - navigation.instant # - navigation.prune - navigation.sections - # - navigation.tabs + - navigation.tabs # - navigation.tabs.sticky - navigation.top - navigation.tracking @@ -58,11 +58,8 @@ theme: # Plugins -# TODO: enable when it's released to general public -# https://squidfunk.github.io/mkdocs-material/insiders/#12000-piri-piri -# plugins: - # - blog: - # post_date_format: full +plugins: + - mkdocstrings # Extras @@ -111,5 +108,7 @@ nav: - Documentation: Installation: documentation/installation.md Usage: documentation/usage.md + - Reference: + Schema: reference/schema.md - Contributing: Development: contributing/development.md diff --git a/pyproject.toml b/pyproject.toml index e760ed2..fab531b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dev = [ "pytest-dotenv", "pytest-timeout", "pytest-lazy-fixture", + "mkdocstrings[python]", "mkdocs-material", ] diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/models/test_schema.py b/tests/models/test_schema.py new file mode 100644 index 0000000..4a69331 --- /dev/null +++ b/tests/models/test_schema.py @@ -0,0 +1,6 @@ +from dpspecs import Schema + + +def test_error(): + schema = Schema(fields=[]) + assert schema.fields == []