From db1bd6a2eae5899cdcb53093c6283a6bf31b1c89 Mon Sep 17 00:00:00 2001 From: Justinas Kenstavicius Date: Fri, 10 Nov 2023 09:49:59 +0200 Subject: [PATCH] 204 refactored BaseQueryBuilder metadata transfer --- spinta/backends/postgresql/commands/query.py | 13 ++++++------ spinta/backends/postgresql/commands/read.py | 6 +++--- spinta/commands/read.py | 13 +++++++----- .../datasets/backends/sql/commands/query.py | 10 ++++----- spinta/datasets/backends/sql/commands/read.py | 6 +++--- spinta/manifests/tabular/helpers.py | 4 ++-- spinta/ufuncs/basequerybuilder/components.py | 21 +++++++++++++------ 7 files changed, 43 insertions(+), 30 deletions(-) diff --git a/spinta/backends/postgresql/commands/query.py b/spinta/backends/postgresql/commands/query.py index 094dfd52a..5e55e066e 100644 --- a/spinta/backends/postgresql/commands/query.py +++ b/spinta/backends/postgresql/commands/query.py @@ -23,7 +23,8 @@ from spinta.exceptions import UnknownMethod from spinta.exceptions import FieldNotInResource from spinta.components import Model, Property, Action, Page, UrlParams -from spinta.ufuncs.basequerybuilder.components import BaseQueryBuilder, QueryPage, merge_with_page_sort, merge_with_page_limit, merge_with_page_selected_list +from spinta.ufuncs.basequerybuilder.components import BaseQueryBuilder, QueryPage, merge_with_page_sort, \ + merge_with_page_limit, merge_with_page_selected_list, QueryParams from spinta.utils.data import take from spinta.types.datatype import DataType, ExternalRef, Inherit, BackRef, Time, ArrayBackRef from spinta.types.datatype import Array @@ -45,7 +46,7 @@ class PgQueryBuilder(BaseQueryBuilder): backend: PostgreSQL - def init(self, params: UrlParams, backend: PostgreSQL, table: sa.Table): + def init(self, backend: PostgreSQL, table: sa.Table, params: QueryParams = None): result = self( backend=backend, table=table, @@ -58,9 +59,9 @@ def init(self, params: UrlParams, backend: PostgreSQL, table: sa.Table): offset=None, aggregate=False, page=QueryPage(), - group_by=[] + group_by=[], ) - result.parse_params(params) + result.init_query_params(params) return result def build(self, where): @@ -478,7 +479,7 @@ def select(env, dtype): def select(env, dtype): default_langs = env.context.get('config').languages table = env.backend.get_table(env.model) - column = env.backend.get_column(table, dtype.prop, langs=env.lang_priority, default_langs=default_langs) + column = env.backend.get_column(table, dtype.prop, langs=env.query_params.lang_priority, default_langs=default_langs) return Selected(column, dtype.prop) @@ -530,7 +531,7 @@ def select(env, dtype, leaf): @ufunc.resolver(PgQueryBuilder, Ref) def select(env, dtype): uri = dtype.model.uri_prop - if env.prioritize_uri and uri is not None: + if env.query_params.prioritize_uri and uri is not None: fpr = ForeignProperty(None, dtype.prop, dtype.model.properties['_id']) table = env.get_joined_table(fpr) column = table.c[uri.place] diff --git a/spinta/backends/postgresql/commands/read.py b/spinta/backends/postgresql/commands/read.py index 397811cb8..44b8a84c4 100644 --- a/spinta/backends/postgresql/commands/read.py +++ b/spinta/backends/postgresql/commands/read.py @@ -9,7 +9,7 @@ from spinta.exceptions import NotFoundError from spinta.exceptions import ItemDoesNotExist from spinta.backends.postgresql.components import PostgreSQL -from spinta.ufuncs.basequerybuilder.components import get_page_values +from spinta.ufuncs.basequerybuilder.components import get_page_values, QueryParams from spinta.ufuncs.helpers import merge_formulas from spinta.utils.nestedstruct import flat_dicts_to_nested from spinta.backends.postgresql.commands.query import PgQueryBuilder @@ -43,7 +43,7 @@ def getall( *, query: Expr = None, default_expand: bool = True, - params: UrlParams = None, + params: QueryParams = None, **kwargs ) -> Iterator[ObjectData]: assert isinstance(query, (Expr, type(None))), query @@ -57,7 +57,7 @@ def getall( builder = PgQueryBuilder(context) builder.update(model=model) table = backend.get_table(model) - env = builder.init(params, backend, table) + env = builder.init(backend, table, params) expr = env.resolve(query) where = env.execute(expr) qry = env.build(where) diff --git a/spinta/commands/read.py b/spinta/commands/read.py index f9b046881..42f32031a 100644 --- a/spinta/commands/read.py +++ b/spinta/commands/read.py @@ -28,7 +28,8 @@ from spinta.exceptions import ItemDoesNotExist from spinta.types.datatype import DataType from spinta.typing import ObjectData -from spinta.ufuncs.basequerybuilder.components import get_allowed_page_property_types +from spinta.ufuncs.basequerybuilder.components import get_allowed_page_property_types, QueryParams, \ + update_query_with_url_params from spinta.ufuncs.basequerybuilder.ufuncs import add_page_expr from spinta.utils.data import take from spinta.utils.encoding import encode_page_values @@ -57,6 +58,8 @@ async def getall( expr = urlparams_to_expr(params, add_count=False) else: expr = urlparams_to_expr(params) + query_params = QueryParams() + update_query_with_url_params(query_params, params) accesslog: AccessLog = context.get('accesslog') accesslog.request( # XXX: Read operations does not have a transaction, but it @@ -70,12 +73,12 @@ async def getall( rows = [] else: if is_page_enabled: - rows = get_page(context, model, backend, copy_page, expr, params.limit, default_expand=False, params=params) + rows = get_page(context, model, backend, copy_page, expr, params.limit, default_expand=False, params=query_params) else: if backend.support_expand: - rows = commands.getall(context, model, backend, params=params, query=expr, default_expand=False) + rows = commands.getall(context, model, backend, params=query_params, query=expr, default_expand=False) else: - rows = commands.getall(context, model, backend, params=params, query=expr) + rows = commands.getall(context, model, backend, params=query_params, query=expr) if params.count: # XXX: Quick and dirty hack. Functions should be handled properly. @@ -202,7 +205,7 @@ def get_page( expr: Expr, limit: Optional[int] = None, default_expand: bool = True, - params: UrlParams = None, + params: QueryParams = None, ) -> Iterator[ObjectData]: config = context.get('config') page_size = config.push_page_size diff --git a/spinta/datasets/backends/sql/commands/query.py b/spinta/datasets/backends/sql/commands/query.py index 846b59355..34a81cfbd 100644 --- a/spinta/datasets/backends/sql/commands/query.py +++ b/spinta/datasets/backends/sql/commands/query.py @@ -40,7 +40,7 @@ from spinta.types.file.components import FileData from spinta.types.text.components import Text from spinta.ufuncs.basequerybuilder.components import BaseQueryBuilder, QueryPage, merge_with_page_selected_list, \ - merge_with_page_sort, merge_with_page_limit + merge_with_page_sort, merge_with_page_limit, QueryParams from spinta.ufuncs.components import ForeignProperty from spinta.core.ufuncs import Unresolved from spinta.utils.data import take @@ -131,7 +131,7 @@ class SqlQueryBuilder(BaseQueryBuilder): selected: Dict[str, Selected] = None params: ResolvedParams - def init(self, params: UrlParams, backend: Sql, table: sa.Table): + def init(self, backend: Sql, table: sa.Table, params: QueryParams = None): result = self( backend=backend, table=table, @@ -142,9 +142,9 @@ def init(self, params: UrlParams, backend: Sql, table: sa.Table): sort=[], limit=None, offset=None, - page=QueryPage() + page=QueryPage(), ) - result.parse_params(params) + result.init_query_params(params) return result def build(self, where): @@ -674,7 +674,7 @@ def select(env: SqlQueryBuilder, dtype: DataType) -> Selected: def select(env, dtype): default_langs = env.context.get('config').languages table = env.backend.get_table(env.model) - column = env.backend.get_column(table, dtype.prop, langs=env.lang_priority, default_langs=default_langs) + column = env.backend.get_column(table, dtype.prop, langs=env.query_params.lang_priority, default_langs=default_langs) return Selected(env.add_column(column), dtype.prop) diff --git a/spinta/datasets/backends/sql/commands/read.py b/spinta/datasets/backends/sql/commands/read.py index 55885cc98..b5bb9373d 100644 --- a/spinta/datasets/backends/sql/commands/read.py +++ b/spinta/datasets/backends/sql/commands/read.py @@ -23,7 +23,7 @@ from spinta.exceptions import ValueNotInEnum, BackendNotGiven from spinta.types.datatype import PrimaryKey from spinta.types.datatype import Ref -from spinta.ufuncs.basequerybuilder.components import get_page_values +from spinta.ufuncs.basequerybuilder.components import get_page_values, QueryParams from spinta.ufuncs.helpers import merge_formulas from spinta.utils.nestedstruct import flat_dicts_to_nested from spinta.utils.schema import NA @@ -77,7 +77,7 @@ def getall( backend: Sql, *, query: Expr = None, - params: UrlParams = None, + params: QueryParams = None, **kwargs ) -> Iterator[ObjectData]: conn = context.get(f'transaction.{backend.name}') @@ -91,7 +91,7 @@ def getall( for model_params in iterparams(context, model): table = model.external.name.format(**model_params) table = backend.get_table(model, table) - env = builder.init(params, backend, table) + env = builder.init(backend, table, params) env.update(params=model_params) expr = env.resolve(query) where = env.execute(expr) diff --git a/spinta/manifests/tabular/helpers.py b/spinta/manifests/tabular/helpers.py index 21f7812a3..cf9befa70 100644 --- a/spinta/manifests/tabular/helpers.py +++ b/spinta/manifests/tabular/helpers.py @@ -761,7 +761,7 @@ def _string_datatype_handler(reader: PropertyReader, row: dict): ) ) if row['ref']: - pass + new_data['enum'] = row['ref'] if dataset or row['source']: new_data['external'] = { 'name': row['source'], @@ -829,7 +829,7 @@ def _text_datatype_handler(reader: PropertyReader, row: dict): ) ) if row['ref']: - pass + new_data['enum'] = row['ref'] if dataset or row['source']: new_data['external'] = { 'name': row['source'], diff --git a/spinta/ufuncs/basequerybuilder/components.py b/spinta/ufuncs/basequerybuilder/components.py index 17fb29aa3..39af673df 100644 --- a/spinta/ufuncs/basequerybuilder/components.py +++ b/spinta/ufuncs/basequerybuilder/components.py @@ -21,16 +21,20 @@ def __init__(self): self.page_ = Page() +class QueryParams: + prioritize_uri: bool = False + lang_priority: List[str] = None + + class BaseQueryBuilder(Env): page: QueryPage expand: List[Property] = None - prioritize_uri: bool = False - lang_priority: List[str] = None + query_params: QueryParams = None - def parse_params(self, params: UrlParams): - if params is not None: - self.lang_priority = params.accept_langs - self.prioritize_uri = params.fmt.prioritize_uri + def init_query_params(self, params: QueryParams): + if params is None: + params = QueryParams() + self.query_params = params class LoadBuilder(Env): @@ -136,3 +140,8 @@ def merge_with_page_limit(limit: int, page: QueryPage): return limit return page.size return limit + + +def update_query_with_url_params(query_params: QueryParams, url_params: UrlParams): + query_params.prioritize_uri = url_params.fmt.prioritize_uri + query_params.lang_priority = url_params.accept_langs