113 Read manifest from database

notes/manifests/internal.sh

@@ -0,0 +1,86 @@
+# notes/docker.sh           Start docker compose
+# notes/postgres.sh         Reset database
+
+INSTANCE=manifests/internal
+DATASET=$INSTANCE
+BASEDIR=var/instances/$INSTANCE
+
+mkdir -p $BASEDIR
+
+export SPINTA_CONFIG=$BASEDIR/config.yml
+
+cat $BASEDIR/config.yml
+cat > $BASEDIR/config.yml <<EOF
+env: production
+data_path: $PWD/$BASEDIR
+default_auth_client: default
+
+keymaps:
+  default:
+    type: sqlalchemy
+    dsn: sqlite:///$PWD/$BASEDIR/keymap.db
+
+backends:
+  default:
+    type: postgresql
+    dsn: postgresql://admin:admin123@localhost:54321/spinta
+
+manifest: default
+manifests:
+  default:
+    type: internal
+    path: postgresql://admin:admin123@localhost:54321/spinta
+    backend: default
+    keymap: default
+    mode: internal
+
+accesslog:
+  type: file
+  file: $PWD/$BASEDIR/accesslog.json
+EOF
+
+poetry run spinta --tb=native bootstrap
+#| Traceback (most recent call last):
+#|   File "spinta/cli/migrate.py", line 27, in bootstrap
+#|     store = prepare_manifest(context, ensure_config_dir=True)
+#|   File "spinta/cli/helpers/store.py", line 132, in prepare_manifest
+#|     store = load_manifest(
+#|   File "spinta/cli/helpers/store.py", line 116, in load_manifest
+#|     commands.load(
+#|   File "spinta/manifests/internal_sql/commands/load.py", line 49, in load
+#|     load_manifest_nodes(context, manifest, schemas)
+#|   File "spinta/manifests/helpers.py", line 134, in load_manifest_nodes
+#|     for eid, schema in schemas:
+#|   File "spinta/manifests/internal_sql/helpers.py", line 36, in read_schema
+#|     yield from _read_all_sql_manifest_rows(path, conn)
+#|   File "spinta/manifests/internal_sql/helpers.py", line 75, in _read_all_sql_manifest_rows
+#|     rows = conn.execute(stmt)
+#| ProgrammingError: (psycopg2.errors.UndefinedTable) relation "_manifest" does not exist
+# FIXME: `spinta bootstrap` should create _manifest table.
+
+psql -h localhost -p 54321 -U admin spinta -c '\dt public.*'
+
+cat > $BASEDIR/manifest.txt <<EOF
+d | r | b | m | property    | type    | ref     | access
+$DATASET                    |         |         |
+  |   |   | Country         |         | id      |
+  |   |   |   | id          | integer |         | open
+  |   |   |   | name        | string  |         | open
+  |   |   |   |             |         |         |
+  |   |   | City            |         | id      |
+  |   |   |   | id          | integer |         | open
+  |   |   |   | name        | string  |         | open
+  |   |   |   | country     | ref     | Country | open
+EOF
+poetry run spinta copy $BASEDIR/manifest.txt -o postgresql://admin:admin123@localhost:54321/spinta
+poetry run spinta show
+
+psql -h localhost -p 54321 -U admin spinta -c 'select * from _manifest;'
+
+poetry run spinta bootstrap
+
+psql -h localhost -p 54321 -U admin spinta -c '\dt public.*'
+
+# notes/spinta/server.sh    Run migrations
+# notes/spinta/server.sh    Run server
+# notes/spinta/client.sh    Configure client

spinta/cli/inspect.py

@@ -26,8 +26,9 @@
 from spinta.manifests.components import Manifest
 from spinta.manifests.components import ManifestPath
 from spinta.manifests.helpers import get_manifest_from_type, init_manifest
-from spinta.manifests.tabular.helpers import render_tabular_manifest
-from spinta.manifests.tabular.helpers import write_tabular_manifest
+from spinta.manifests.internal_sql.components import InternalSQLManifest
+from spinta.manifests.internal_sql.helpers import write_internal_sql_manifest
+from spinta.manifests.tabular.helpers import write_tabular_manifest, render_tabular_manifest
 from spinta.types.datatype import Ref, DataType, Array, Object, Denorm
 from spinta.utils.naming import Deduplicator
 from spinta.utils.schema import NA
@@ -107,7 +108,10 @@ def inspect(
     manifest.objects['model'] = sorted_models
 
     if output:
-        write_tabular_manifest(output, manifest)
+        if InternalSQLManifest.detect_from_path(output):
+            write_internal_sql_manifest(output, manifest)
+        else:
+            write_tabular_manifest(output, manifest)
     else:
         echo(render_tabular_manifest(manifest))
 

spinta/cli/manifest.py

@@ -12,6 +12,8 @@
 from spinta.components import Context
 from spinta.core.context import configure_context
 from spinta.core.enums import Access
+from spinta.manifests.internal_sql.components import InternalSQLManifest
+from spinta.manifests.internal_sql.helpers import write_internal_sql_manifest
 from spinta.manifests.tabular.components import ManifestColumn
 from spinta.manifests.tabular.components import ManifestRow
 from spinta.manifests.tabular.helpers import datasets_to_tabular
@@ -62,25 +64,65 @@ def copy(
     verbose = True
     if not output:
         verbose = False
+    internal = InternalSQLManifest.detect_from_path(output)
+    if output and internal:
+        rows = _read_and_return_manifest(
+            context,
+            manifests,
+            external=source,
+            access=access,
+            format_names=format_names,
+            order_by=order_by,
+            rename_duplicates=rename_duplicates,
+            verbose=verbose,
+        )
+    else:
+        rows = _read_and_return_rows(
+            context,
+            manifests,
+            external=source,
+            access=access,
+            format_names=format_names,
+            order_by=order_by,
+            rename_duplicates=rename_duplicates,
+            verbose=verbose,
+        )
 
-    rows = _read_csv_files(
+    if output:
+        if internal:
+            write_internal_sql_manifest(output, rows)
+        else:
+            write_tabular_manifest(output, rows)
+    else:
+        echo(render_tabular_manifest_rows(rows, cols))
+
+
+def _read_and_return_manifest(
+    context: Context,
+    manifests: List[str],
+    *,
+    external: bool = True,
+    access: Access = Access.private,
+    format_names: bool = False,
+    order_by: ManifestColumn = None,
+    rename_duplicates: bool = False,
+    verbose: bool = True,
+) -> Iterator[ManifestRow]:
+    context = configure_context(context, manifests)
+    store = load_manifest(
         context,
-        manifests,
-        external=source,
-        access=access,
-        format_names=format_names,
-        order_by=order_by,
         rename_duplicates=rename_duplicates,
+        load_internal=False,
         verbose=verbose,
     )
 
-    if output:
-        write_tabular_manifest(output, rows)
-    else:
-        echo(render_tabular_manifest_rows(rows, cols))
+    if format_names:
+        reformat_names(context, store.manifest)
+
+    return store.manifest
 
 
-def _read_csv_files(
+def _read_and_return_rows(
     context: Context,
     manifests: List[str],
     *,

spinta/components.py

@@ -411,6 +411,13 @@ def get_eid_for_error_context(self):
             return str(self.eid)
 
 
+class ExtraMetaData(Node):
+    id: str = None
+    schema = {
+        'id': {'type': 'string'}
+    }
+
+
 class NamespaceGiven:
     access: str = None
 
@@ -452,7 +459,7 @@ def is_root(self) -> bool:
         return isinstance(self.parent, Manifest)
 
 
-class Base(Node):
+class Base(ExtraMetaData):
     model: Model        # a.base.b - here `model` is `b`
     parent: Model       # a.base.b - here `parent` is `a`
     pk: List[Property]  # a.base.b - list of properties of `a` model
@@ -549,7 +556,7 @@ class PropertyGiven:
     unit: str = None
 
 
-class Property(Node):
+class Property(ExtraMetaData):
     place: str = None  # Dotted property path
     title: str = None
     description: str = None

spinta/config.py

@@ -47,7 +47,8 @@
             'sql': 'spinta.manifests.sql.components:SqlManifest',
             'memory': 'spinta.manifests.memory.components:MemoryManifest',
             'json': 'spinta.manifests.dict.components:JsonManifest',
-            'xml': 'spinta.manifests.dict.components:XmlManifest'
+            'xml': 'spinta.manifests.dict.components:XmlManifest',
+            'internal': 'spinta.manifests.internal_sql.components:InternalSQLManifest',
         },
         'backends': {
             # In memory backends mostly usable in tests

spinta/datasets/components.py

@@ -8,12 +8,11 @@
 from sqlalchemy.engine.base import Engine
 
 from spinta.backends.components import Backend
-from spinta.components import EntryId
+from spinta.components import EntryId, ExtraMetaData
 from spinta.components import Namespace
 from spinta.dimensions.lang.components import LangData
 from spinta.components import MetaData
 from spinta.components import Model
-from spinta.components import Node
 from spinta.components import Property
 from spinta.core.enums import Access
 from spinta.core.ufuncs import Expr
@@ -22,6 +21,7 @@
 from spinta.manifests.components import Manifest
 from spinta.types.owner import Owner
 from spinta.types.project import Project
+from spinta.utils.schema import NA
 
 
 class DatasetGiven:
@@ -96,7 +96,7 @@ class ExternalBackend(Backend):
     schema: sa.MetaData = None
 
 
-class External(Node):
+class External(ExtraMetaData):
     pass
 
 
@@ -113,7 +113,7 @@ class Resource(External):
     level: Level
     access: Access
     external: str
-    prepare: str
+    prepare: Expr
     models: Dict[str, Model]
     given: ResourceGiven
     lang: LangData = None
@@ -122,7 +122,7 @@ class Resource(External):
     schema = {
         'type': {'type': 'string'},
         'dataset': {'parent': True},
-        'prepare': {'type': 'spyna'},
+        'prepare': {'type': 'spyna', 'default': NA},
 
         # Backend name specified in `ref` column, points to previously defined
         # backend or to a configured stored backend.
@@ -151,6 +151,7 @@ class Resource(External):
         'title': {'type': 'string'},
         'description': {'type': 'string'},
         'comments': {},
+        'lang': {'type': 'object'},
     }
 
     def __init__(self):
@@ -186,7 +187,7 @@ class Entity(External):
         'dataset': {'type': 'ref', 'ref': 'context.nodes.dataset'},
         'resource': {'type': 'ref', 'ref': 'dataset.resources'},
         'name': {'type': 'string', 'default': None},
-        'prepare': {'type': 'spyna', 'default': None},
+        'prepare': {'type': 'spyna', 'default': NA},
         'params': {
             'type': 'array',
             'items': {'type': 'object'},
@@ -206,10 +207,10 @@ class Entity(External):
 class Attribute(External):
     prop: Property          # property
     name: str               # property.source
-    prepare: Expr = None    # property.prepare
+    prepare: Expr = NA    # property.prepare
 
     schema = {
         'prop': {'parent': True},
         'name': {'default': None},
-        'prepare': {'type': 'spyna', 'default': None},
+        'prepare': {'type': 'spyna', 'default': NA},
     }

spinta/dimensions/enum/components.py

@@ -2,8 +2,7 @@
 
 from typing import Dict
 
-from spinta.components import Model
-from spinta.components import Node
+from spinta.components import Model, ExtraMetaData
 from spinta.core.enums import Access
 from spinta.core.ufuncs import Env
 from spinta.core.ufuncs import Expr
@@ -19,7 +18,7 @@ class EnumValueGiven:
     access: str = None
 
 
-class EnumItem(Node):
+class EnumItem(ExtraMetaData):
     source: str
     prepare: Expr
     access: Access = None

spinta/manifests/internal_sql/commands/bootstrap.py

@@ -0,0 +1,10 @@
+from spinta import commands
+from spinta.components import Context
+from spinta.manifests.internal_sql.components import InternalSQLManifest
+
+
+@commands.bootstrap.register(Context, InternalSQLManifest)
+def bootstrap(context: Context, manifest: InternalSQLManifest):
+    store = context.get('store')
+    for backend in store.backends.values():
+        commands.bootstrap(context, backend)

spinta/manifests/internal_sql/commands/configure.py

@@ -0,0 +1,13 @@
+from typing import Optional
+
+from spinta import commands
+from spinta.components import Context
+from spinta.core.config import RawConfig
+from spinta.manifests.internal_sql.components import InternalSQLManifest
+
+
+@commands.configure.register(Context, InternalSQLManifest)
+def configure(context: Context, manifest: InternalSQLManifest):
+    rc: RawConfig = context.get('rc')
+    path: Optional[str] = rc.get('manifests', manifest.name, 'path')
+    manifest.path = path