Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

476 Make sure schema and views works with inspect #502

Merged
merged 7 commits into from
Nov 14, 2023
Merged

476 Make sure schema and views works with inspect #502

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7358e07
476 added prepare and views to manifest loading, fixed inspect bugs
adp-atea Aug 3, 2023
d9b4b8c
476 Fixed connect for backwards compatability
adp-atea Aug 4, 2023
fa0b1ae
476 Fixed view test
adp-atea Aug 4, 2023
42a7c96
476 Another test fix
adp-atea Aug 4, 2023
7042cb1
476 Fixed for linux test
adp-atea Aug 4, 2023
c2ecad4
476 Removed old leftover frictionless implementations
adp-atea Aug 4, 2023
91f134d
Testing notes
sirex Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 35 additions & 19 deletions notes/cli/inspect.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@ CREATE TABLE IF NOT EXISTS cities (
country_id INTEGER,
FOREIGN KEY (country_id) REFERENCES countries (id)
);
CREATE VIEW cities_view AS
SELECT id, name FROM cities;
EOF
sqlite3 $BASEDIR/db.sqlite ".schema"

unset SPINTA_CONFIG
test -f $BASEDIR/config.yml && echo "Remove $BASEDIR/config.yml"
test -f $BASEDIR/config.yml && rm $BASEDIR/config.yml

poetry run spinta config manifest
#| Origin Name Value
Expand Down Expand Up @@ -55,6 +57,12 @@ poetry run spinta show $BASEDIR/manifest.csv
#| | | | | | continent_id | integer | | continent_id | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | dbsqlite/views | | | | | | | | |
#| | | resource1 | sql | | sqlite:///var/instances/cli/inspect/db.sqlite | | | | | |
#| | | | | | | | | | |
#| | | | | CitiesView | | | cities_view | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |


cat > $BASEDIR/manifest.txt <<EOF
Expand Down Expand Up @@ -107,6 +115,10 @@ poetry run spinta show $BASEDIR/manifest.csv
#| | | | | Continents | | id | continents | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | | | | | | | | | |
#| | | | | CitiesView | | | cities_view | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |


cat > $BASEDIR/manifest.txt <<EOF
Expand Down Expand Up @@ -143,24 +155,28 @@ poetry run spinta inspect $BASEDIR/manifest.csv -o $BASEDIR/manifest.csv
#| Loading InlineManifest manifest default

poetry run spinta show $BASEDIR/manifest.csv
#| id | d | r | b | m | property | type | ref | source | prepare | level | access | uri | title | description
#| | datasets/examplle | | | | | | | | |
#| | | db | sql | | sqlite:///var/instances/cli/inspect/db.sqlite | | | | | |
#| | | | | | | | | | |
#| | | | | City | | id | cities | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | | | | | country | ref | Country | country_id | | | | | |
#| | | | | | | | | | |
#| | | | | Country | | id | countries | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | | | | | continent | integer | | continent_id | | | | | |
#| | | | | | population | integer | | population | | | | | |
#| | | | | | | | | | |
#| | | | | Continent | | id | continents | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| id | d | r | b | m | property | type | ref | source | prepare | level | access | uri | title | description
#| | datasets/examplle | | | | | | | | |
#| | | db | sql | | sqlite:///var/instances/cli/inspect/db.sqlite | | | | | |
#| | | | | | | | | | |
#| | | | | City | | id | cities | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | | | | | country | ref | Country | country_id | | | | | |
#| | | | | | | | | | |
#| | | | | Country | | id | countries | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | | | | | continent | ref | Continent | continent_id | | | | | |
#| | | | | | population | integer | | population | | | | | |
#| | | | | | | | | | |
#| | | | | Continent | | id | continents | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |
#| | | | | | | | | | |
#| | | | | CitiesView | | | cities_view | | | | | |
#| | | | | | id | integer | | id | | | | | |
#| | | | | | name | string | | name | | | | | |


sqlite3 $BASEDIR/db.sqlite <<'EOF'
Expand Down
44 changes: 19 additions & 25 deletions spinta/cli/inspect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,12 @@
from spinta.cli.helpers.store import load_manifest
from spinta.components import Context, Property, Node, Namespace
from spinta.components import Mode
from spinta.core.config import RawConfig
from spinta.core.config import ResourceTuple
from spinta.core.config import parse_resource_args
from spinta.core.context import configure_context
from spinta.datasets.components import Dataset, Resource, Attribute, Entity, ExternalBackend
from spinta.datasets.components import Dataset, Resource, ExternalBackend
from spinta.dimensions.prefix.components import UriPrefix
from spinta.manifests.components import Manifest
from spinta.manifests.components import ManifestPath
from spinta.manifests.helpers import get_manifest_from_type, init_manifest
from spinta.manifests.tabular.helpers import render_tabular_manifest
from spinta.manifests.tabular.helpers import write_tabular_manifest
from spinta.types.datatype import Ref, DataType, Array, Object, Denorm
Expand Down Expand Up @@ -67,17 +64,19 @@ def inspect(
priority = 'manifest'
has_manifest_priority = priority == 'manifest'
resources = parse_resource_args(*resource, formula)

paths = None
if manifest:
paths = [manifest]

context = configure_context(
ctx.obj,
[manifest] if manifest else None,
paths,
mode=Mode.external,
backend=backend,
backend=backend
)
store = load_manifest(context, ensure_config_dir=True)
old = store.manifest
manifest = Manifest()
init_manifest(context, manifest, 'inspect')
commands.merge(context, manifest, manifest, old, has_manifest_priority)

if not resources:
resources = []
Expand All @@ -91,11 +90,11 @@ def inspect(

if resources:
for resource in resources:
_merge(context, manifest, manifest, resource, has_manifest_priority)
_merge(context, old, old, resource, has_manifest_priority)

# Sort models for render
sorted_models = {}
for key, model in manifest.models.items():
for key, model in old.models.items():
if key not in sorted_models.keys():
if model.external and model.external.resource:
resource = model.external.resource
Expand All @@ -104,31 +103,26 @@ def inspect(
sorted_models[resource_key] = resource_model
else:
sorted_models[key] = model
manifest.objects['model'] = sorted_models
old.objects['model'] = sorted_models

if output:
write_tabular_manifest(output, manifest)
write_tabular_manifest(output, old)
else:
echo(render_tabular_manifest(manifest))
echo(render_tabular_manifest(old))


def _merge(context: Context, manifest: Manifest, old: Manifest, resource: ResourceTuple, has_manifest_priority: bool):
rc: RawConfig = context.get('rc')
Manifest_ = get_manifest_from_type(rc, resource.type)
path = ManifestPath(type=Manifest_.type, path=resource.external)
context = configure_context(context, [path], mode=Mode.external)
store = load_manifest(context)
new_context = configure_context(
context,
manifests=[resource],
mode=Mode.external)
store = load_manifest(new_context)
new = store.manifest
commands.merge(context, manifest, old, new, has_manifest_priority)
commands.merge(new_context, manifest, old, new, has_manifest_priority)


@commands.merge.register(Context, Manifest, Manifest, Manifest, bool)
def merge(context: Context, manifest: Manifest, old: Manifest, new: Manifest, has_manifest_priority: bool) -> None:
resource_list = []
for ds in new.datasets.values():
for res in ds.resources.values():
resource_list.append(_resource_source_key(res))

backends = zipitems(
old.backends.values(),
new.backends.values(),
Expand Down
12 changes: 10 additions & 2 deletions spinta/core/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
from ruamel.yaml import YAML
import pkg_resources as pres

from spinta import spyna
from spinta.components import Mode
from spinta.core.ufuncs import asttoexpr
from spinta.utils.imports import importstr
from spinta.utils.schema import NA

Expand Down Expand Up @@ -546,12 +548,14 @@ def parse_resource_args(

def _parse_manifest_path(
rc: RawConfig,
path: Union[str, ManifestPath],
path: Union[str, ManifestPath, ResourceTuple],
) -> ManifestPath:
from spinta.manifests.components import ManifestPath
if isinstance(path, ManifestPath):
return path
from spinta.manifests.helpers import detect_manifest_from_path
if isinstance(path, ResourceTuple):
path = path.external
Manifest_ = detect_manifest_from_path(rc, path)
return ManifestPath(type=Manifest_.type, path=path)

Expand Down Expand Up @@ -579,7 +583,7 @@ def _get_resource_config(

def configure_rc(
rc: RawConfig,
manifests: List[Union[str, ManifestPath]] = None,
manifests: List[Union[str, ManifestPath, ResourceTuple]] = None,
*,
mode: Mode = Mode.internal,
backend: str = None,
Expand Down Expand Up @@ -624,6 +628,10 @@ def configure_rc(
'path': manifest.path,
'file': manifest.file,
}
if isinstance(path, ResourceTuple) and path.prepare:
parsed = spyna.parse(path.prepare)
converted = asttoexpr(parsed)
config[f'manifests.{manifest_name}']['prepare'] = converted
sync.append(manifest_name)

if resources:
Expand Down
Empty file removed spinta/datasets/backends/frictionless/__init__.py
View file
Open in desktop
Empty file.
Empty file removed spinta/datasets/backends/frictionless/commands/__init__.py
View file
Open in desktop
Empty file.
11 changes: 0 additions & 11 deletions spinta/datasets/backends/frictionless/commands/load.py
View file
Open in desktop

This file was deleted.

8 changes: 0 additions & 8 deletions spinta/datasets/backends/frictionless/commands/wait.py
View file
Open in desktop

This file was deleted.

9 changes: 0 additions & 9 deletions spinta/datasets/backends/frictionless/commands/wipe.py
View file
Open in desktop

This file was deleted.

5 changes: 0 additions & 5 deletions spinta/datasets/backends/frictionless/components.py
View file
Open in desktop

This file was deleted.

Loading