Merge pull request #1041 from atviriduomenys/973-prideti-object-tipo-…

…palaikyma-isoriniam-sql-backendu-0.1 973 prideti object tipo palaikyma isoriniam sql backendu 0.1
atviriduomenys · Dec 19, 2024 · 14db44a · 14db44a
2 parents 2c7f18a + ac6399e
commit 14db44a
Show file tree

Hide file tree

Showing 4 changed files with 212 additions and 10 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -4,6 +4,11 @@ Changes
 0.1.82 (unreleased)
 ===================
 
+New features:
+
+- Added support for `Object` type with `external` `Sql` `backend` (`#973`_).
+
+  .. _#973: https://github.com/atviriduomenys/spinta/issues/973
 
 0.1.81 (2024-12-17)
 ===================

diff --git a/spinta/datasets/backends/sql/ufuncs/query/ufuncs.py b/spinta/datasets/backends/sql/ufuncs/query/ufuncs.py
@@ -1,32 +1,28 @@
 from __future__ import annotations
 
-from typing import Any, Tuple
+from typing import Any
 from typing import List
 from typing import Tuple
 from typing import TypeVar
 from typing import Union
 from typing import overload
-from typing import Dict
 
 import sqlalchemy as sa
 from sqlalchemy.sql.functions import Function
 
-from spinta import exceptions
 from spinta.auth import authorized
 from spinta.components import Action, Page
-from spinta.components import Model
 from spinta.components import Property
 from spinta.core.ufuncs import Bind, GetAttr
 from spinta.core.ufuncs import Expr
 from spinta.core.ufuncs import Negative
 from spinta.core.ufuncs import Unresolved
 from spinta.core.ufuncs import ufunc
 from spinta.datasets.backends.sql.helpers import dialect_specific_desc, dialect_specific_asc
-from spinta.datasets.backends.sql.ufuncs.components import Selected
 from spinta.datasets.backends.sql.ufuncs.query.components import SqlQueryBuilder
 from spinta.dimensions.enum.helpers import prepare_enum_value
 from spinta.exceptions import PropertyNotFound, SourceCannotBeList
-from spinta.types.datatype import DataType, Denorm
+from spinta.types.datatype import DataType, Denorm, Object
 from spinta.types.datatype import PrimaryKey
 from spinta.types.datatype import Ref
 from spinta.types.datatype import String
@@ -36,7 +32,6 @@
 from spinta.ufuncs.basequerybuilder.components import LiteralProperty, Selected
 from spinta.ufuncs.basequerybuilder.helpers import get_language_column, process_literal_value
 from spinta.ufuncs.basequerybuilder.ufuncs import Star
-from spinta.ufuncs.basequerybuilder.ufuncs import ResultProperty, NestedProperty, ReservedProperty
 from spinta.ufuncs.components import ForeignProperty
 from spinta.utils.data import take
 from spinta.utils.itertools import flatten
@@ -324,6 +319,7 @@ def select(env: SqlQueryBuilder, expr: Expr):
 def select(env, column):
     return Selected(env.add_column(column))
 
+
 @ufunc.resolver(SqlQueryBuilder, Bind)
 def select(env: SqlQueryBuilder, item: Bind, *, nested: bool = False):
     if item.name == '_page':
@@ -393,7 +389,7 @@ def select(env: SqlQueryBuilder, prop: Property) -> Selected:
             # Reserved properties never have external source.
             result = env.call('select', prop.dtype)
         elif not prop.dtype.requires_source:
-            # Some DataTypes might have children that have source instead of themselves, like: Text
+            # Some DataTypes might have children that have source instead of themselves, like: Text, Object
             result = env.call('select', prop.dtype)
         elif prop.dtype.inherited:
             # Some DataTypes might be inherited, or hidden, so we need to go through them in case they can be joined
@@ -418,6 +414,16 @@ def select(env: SqlQueryBuilder, dtype: DataType) -> Selected:
     )
 
 
+@ufunc.resolver(SqlQueryBuilder, Object)
+def select(env: SqlQueryBuilder, dtype: Object) -> Selected:
+    prep = {}
+    for prop in take(dtype.properties).values():
+        sel = env.call('select', prop)
+        if sel is not None:
+            prep[prop.name] = sel
+    return Selected(prop=dtype.prop, prep=prep)
+
+
 @ufunc.resolver(SqlQueryBuilder, Ref)
 def select(env: SqlQueryBuilder, dtype: Ref) -> Selected:
     table = env.backend.get_table(env.model)

diff --git a/spinta/types/datatype.py b/spinta/types/datatype.py
@@ -272,6 +272,7 @@ class Object(DataType):
     }
 
     properties: Dict[str, Property] = None
+    requires_source = False
 
     def load(self, value: Any):
         if value is None or value is NA:

diff --git a/tests/datasets/test_sql.py b/tests/datasets/test_sql.py
@@ -132,7 +132,7 @@ def geodb_denorm():
         db.write('CITY', [
             {'code': 'VLN', 'name': 'Vilnius', 'country': 'LT', 'countryYear': 1204, 'countryName': 'Lietuva',
              'planetName': 'Zeme'},
-            {'code': 'RYG', 'name': 'Ryga', 'country': 'LV', 'countryYear': 1408, 'countryName': 'Riga',
+            {'code': 'RYG', 'name': 'Ryga', 'country': 'LV', 'countryYear': 1408, 'countryName': 'Latvia',
              'planetName': 'Marsas'},
             {'code': 'TLN', 'name': 'Talin', 'country': 'EE', 'countryYear': 1784, 'countryName': 'Estija',
              'planetName': 'Jupiteris'},
@@ -2973,7 +2973,7 @@ def test_advanced_denorm(context, rc, tmp_path, geodb_denorm):
         'code': 'RYG',
         'name': 'Ryga',
         'country.name': 'Latvia',
-        'country.code': 'Riga',
+        'country.code': 'Latvia',
         'country.year': 1408,
         'country.planet.name': 'Mars',
         'country.planet.code': 'Marsas'
@@ -3467,3 +3467,193 @@ def test_non_pk_external_ref_only_literal(context, rc, tmp_path, sqlite):
         'lt.lang': "lt",
         'name_lt': "Vilniaus miestas"
     }]
+
+
+def test_object(context, rc, tmp_path, geodb_denorm):
+    create_tabular_manifest(context, tmp_path / 'manifest.csv', striptable('''
+    d | r | m | property            | type    | ref     | source       | prepare | access | level
+    datasets/object                 |         |         |              |         |        |
+      | rs                          | sql     |         |              |         |        |
+      |   | City                    |         | code    | CITY         |         | open   |
+      |   |   | code                | string  |         | code         |         |        |
+      |   |   | name                | string  |         | name         |         |        |
+      |   |   | country             | object  |         |              |         |        |
+      |   |   | country.name        | string  |         | countryName  |         |        |
+      |   |   | country.year        | integer |         | countryYear  |         |        |
+    '''))
+
+    app = create_client(rc, tmp_path, geodb_denorm)
+
+    resp = app.get('/datasets/object/City')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'country.name': 'Latvia',
+        'country.year': 1408,
+    }, {
+        'code': 'TLN',
+        'name': 'Talin',
+        'country.name': 'Estija',
+        'country.year': 1784,
+    }, {
+        'code': 'VLN',
+        'name': 'Vilnius',
+        'country.name': 'Lietuva',
+        'country.year': 1204,
+    }]
+
+
+def test_nested_object(context, rc, tmp_path, geodb_denorm):
+    create_tabular_manifest(context, tmp_path / 'manifest.csv', striptable('''
+    d | r | m | property            | type    | ref     | source       | prepare | access | level
+    datasets/object/nested          |         |         |              |         |        |
+      | rs                          | sql     |         |              |         |        |
+      |   | City                    |         | code    | CITY         |         | open   |
+      |   |   | code                | string  |         | code         |         |        |
+      |   |   | name                | string  |         | name         |         |        |
+      |   |   | c                   | object  |         |              |         |        |
+      |   |   | c.country           | object  |         |              |         |        |
+      |   |   | c.country.name      | string  |         | countryName  |         |        |
+      |   |   | c.country.year      | integer |         | countryYear  |         |        |
+    '''))
+
+    app = create_client(rc, tmp_path, geodb_denorm)
+
+    resp = app.get('/datasets/object/nested/City')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'c.country.name': 'Latvia',
+        'c.country.year': 1408,
+    }, {
+        'code': 'TLN',
+        'name': 'Talin',
+        'c.country.name': 'Estija',
+        'c.country.year': 1784,
+    }, {
+        'code': 'VLN',
+        'name': 'Vilnius',
+        'c.country.name': 'Lietuva',
+        'c.country.year': 1204,
+    }]
+
+
+def test_ref_object(context, rc, tmp_path, geodb_denorm):
+    create_tabular_manifest(context, tmp_path / 'manifest.csv', striptable('''
+    d | r | m | property            | type    | ref     | source       | prepare | access | level
+    datasets/object/ref             |         |         |              |         |        |
+      | rs                          | sql     |         |              |         |        |
+      |   | Country                 |         | c       | COUNTRY      |         | open   |
+      |   |   | c                   | string  |         | code         |         |        |
+      |   |   | name                | string  |         | name         |         |        |
+      |   |   | code                | string  |         | code         |         |        |
+      |   | City                    |         | code    | CITY         |         | open   |
+      |   |   | code                | string  |         | code         |         |        |
+      |   |   | name                | string  |         | name         |         |        |
+      |   |   | country             | ref     | Country | country      |         |        | 3
+      |   |   | country.code        |         |         |              |         |        |
+      |   |   | country.meta        | object  |         |              |         |        |
+      |   |   | country.meta.name   | string  |         | countryName  |         |        |
+      |   |   | country.meta.year   | string  |         | countryYear  |         |        |
+    '''))
+
+    app = create_client(rc, tmp_path, geodb_denorm)
+
+    resp = app.get('/datasets/object/ref/City')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'country.c': 'LV',
+        'country.code': 'LV',
+        'country.meta.name': 'Latvia',
+        'country.meta.year': 1408,
+    }, {
+        'code': 'TLN',
+        'name': 'Talin',
+        'country.c': 'EE',
+        'country.code': 'EE',
+        'country.meta.name': 'Estija',
+        'country.meta.year': 1784,
+    }, {
+        'code': 'VLN',
+        'name': 'Vilnius',
+        'country.c': 'LT',
+        'country.code': 'LT',
+        'country.meta.name': 'Lietuva',
+        'country.meta.year': 1204,
+    }]
+
+
+def test_object_filter(context, rc, tmp_path, geodb_denorm):
+    create_tabular_manifest(context, tmp_path / 'manifest.csv', striptable('''
+    d | r | m | property            | type    | ref     | source       | prepare | access | level
+    datasets/object                 |         |         |              |         |        |
+      | rs                          | sql     |         |              |         |        |
+      |   | City                    |         | code    | CITY         |         | open   |
+      |   |   | code                | string  |         | code         |         |        |
+      |   |   | name                | string  |         | name         |         |        |
+      |   |   | country             | object  |         |              |         |        |
+      |   |   | country.name        | string  |         | countryName  |         |        |
+      |   |   | country.year        | integer |         | countryYear  |         |        |
+    '''))
+
+    app = create_client(rc, tmp_path, geodb_denorm)
+
+    resp = app.get('/datasets/object/City?country.year>1300')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'country.name': 'Latvia',
+        'country.year': 1408,
+    }, {
+        'code': 'TLN',
+        'name': 'Talin',
+        'country.name': 'Estija',
+        'country.year': 1784,
+    }]
+
+    resp = app.get('/datasets/object/City?country.year=1408')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'country.name': 'Latvia',
+        'country.year': 1408,
+    }]
+
+
+def test_object_filter_nested(context, rc, tmp_path, geodb_denorm):
+    create_tabular_manifest(context, tmp_path / 'manifest.csv', striptable('''
+    d | r | m | property            | type    | ref     | source       | prepare | access | level
+    datasets/object/nested          |         |         |              |         |        |
+      | rs                          | sql     |         |              |         |        |
+      |   | City                    |         | code    | CITY         |         | open   |
+      |   |   | code                | string  |         | code         |         |        |
+      |   |   | name                | string  |         | name         |         |        |
+      |   |   | c                   | object  |         |              |         |        |
+      |   |   | c.country           | object  |         |              |         |        |
+      |   |   | c.country.name      | string  |         | countryName  |         |        |
+      |   |   | c.country.year      | integer |         | countryYear  |         |        |
+    '''))
+
+    app = create_client(rc, tmp_path, geodb_denorm)
+
+    resp = app.get('/datasets/object/nested/City?c.country.year>1300')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'c.country.name': 'Latvia',
+        'c.country.year': 1408,
+    }, {
+        'code': 'TLN',
+        'name': 'Talin',
+        'c.country.name': 'Estija',
+        'c.country.year': 1784,
+    }]
+
+    resp = app.get('/datasets/object/nested/City?c.country.year=1408')
+    assert listdata(resp, sort='code', full=True) == [{
+        'code': 'RYG',
+        'name': 'Ryga',
+        'c.country.name': 'Latvia',
+        'c.country.year': 1408,
+    }]