import: Options to set dataset metadata

Fixes #102

CHANGELOG.md

@@ -8,6 +8,7 @@ _When adding new entries to the changelog, please include issue/PR numbers where
 
 * Added a `sno meta get` command for viewing dataset metadata.
 * `merge`, `commit`, `init`, `import` commands can now take commit messages as files with `--message=@filename.txt`. This replaces the `sno commit -F` option.
+* `import`: Added `--table-info` option to set dataset metadata, when it can't be autodetected from the source database
 
 ## 0.4.0
 

requirements.txt

@@ -4,16 +4,22 @@
 #
 #    make py-requirements
 #
+attrs==19.3.0             # via jsonschema
 cached-property==1.5.1    # via pygit2
 certifi==2020.6.20        # via -r requirements/requirements.in
 cffi==1.14.0              # via pygit2
 click==7.1.2              # via -r requirements/requirements.in
+importlib-metadata==1.6.1  # via jsonschema
+jsonschema==3.2.0         # via -r requirements/requirements.in
 msgpack==0.6.2            # via -r requirements/requirements.in
 #psycopg2==2.8.5           # via -r requirements/requirements.in
 pycparser==2.20           # via cffi
 #pygit2==1.1.0             # via -r requirements/requirements.in
 pygments==2.6.1           # via -r requirements/requirements.in
+pyrsistent==0.16.0        # via jsonschema
 rtree==0.9.4              # via -r requirements/requirements.in
+six==1.15.0               # via jsonschema, pyrsistent
+zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/dev.txt

@@ -7,7 +7,7 @@
 appnope==0.1.0            # via ipython
 aspectlib==1.5.1          # via -r requirements/test.txt, pytest-benchmark
 atomicwrites==1.4.0       # via -r requirements/test.txt, pytest
-attrs==19.3.0             # via -r requirements/test.txt, pytest
+attrs==19.3.0             # via -r requirements/../requirements.txt, -r requirements/test.txt, jsonschema, pytest
 backcall==0.2.0           # via ipython
 cached-property==1.5.1    # via -r requirements/../requirements.txt, -r requirements/test.txt
 certifi==2020.6.20        # via -r requirements/../requirements.txt, -r requirements/test.txt
@@ -17,12 +17,13 @@ coverage==5.1             # via -r requirements/test.txt, pytest-cov
 decorator==4.4.2          # via ipython, traitlets
 fields==5.0.0             # via -r requirements/test.txt, aspectlib
 gprof2dot==2019.11.30     # via -r requirements/test.txt, pytest-profiling
-html5lib==1.0.1           # via -r requirements/test.txt
-importlib-metadata==1.6.1  # via -r requirements/test.txt, pluggy, pytest
-ipdb==0.13.2              # via -r requirements/dev.in
+html5lib==1.1             # via -r requirements/test.txt
+importlib-metadata==1.6.1  # via -r requirements/../requirements.txt, -r requirements/test.txt, jsonschema, pluggy, pytest
+ipdb==0.13.3              # via -r requirements/dev.in
 ipython-genutils==0.2.0   # via traitlets
 ipython==7.15.0           # via -r requirements/dev.in, ipdb
 jedi==0.17.1              # via ipython
+jsonschema==3.2.0         # via -r requirements/../requirements.txt, -r requirements/test.txt
 lovely-pytest-docker==0.1.0  # via -r requirements/test.txt
 more-itertools==8.4.0     # via -r requirements/test.txt, pytest
 msgpack==0.6.2            # via -r requirements/../requirements.txt, -r requirements/test.txt
@@ -34,23 +35,24 @@ pluggy==0.13.1            # via -r requirements/test.txt, pytest
 prompt-toolkit==3.0.5     # via ipython
 ptyprocess==0.6.0         # via pexpect
 py-cpuinfo==6.0.0         # via -r requirements/test.txt, pytest-benchmark
-py==1.8.2                 # via -r requirements/test.txt, pytest
+py==1.9.0                 # via -r requirements/test.txt, pytest
 pycparser==2.20           # via -r requirements/../requirements.txt, -r requirements/test.txt, cffi
 pygments==2.6.1           # via -r requirements/../requirements.txt, -r requirements/test.txt, ipython
 pyparsing==2.4.7          # via -r requirements/test.txt, packaging
+pyrsistent==0.16.0        # via -r requirements/../requirements.txt, -r requirements/test.txt, jsonschema
 pytest-benchmark[aspect]==3.2.3  # via -r requirements/test.txt
 pytest-cov==2.10.0        # via -r requirements/test.txt
 pytest-helpers-namespace==2019.1.8  # via -r requirements/test.txt
 pytest-profiling==1.7.0   # via -r requirements/test.txt
 pytest-sugar==0.9.3       # via -r requirements/test.txt
 pytest==4.6.11            # via -r requirements/test.txt, lovely-pytest-docker, pytest-benchmark, pytest-cov, pytest-helpers-namespace, pytest-profiling, pytest-sugar
 rtree==0.9.4              # via -r requirements/../requirements.txt, -r requirements/test.txt
-six==1.15.0               # via -r requirements/test.txt, html5lib, packaging, pytest, pytest-profiling, traitlets
+six==1.15.0               # via -r requirements/../requirements.txt, -r requirements/test.txt, html5lib, jsonschema, packaging, pyrsistent, pytest, pytest-profiling, traitlets
 termcolor==1.1.0          # via -r requirements/test.txt, pytest-sugar
 traitlets==4.3.3          # via ipython
-wcwidth==0.2.4            # via -r requirements/test.txt, prompt-toolkit, pytest
+wcwidth==0.2.5            # via -r requirements/test.txt, prompt-toolkit, pytest
 webencodings==0.5.1       # via -r requirements/test.txt, html5lib
-zipp==3.1.0               # via -r requirements/test.txt, importlib-metadata
+zipp==3.1.0               # via -r requirements/../requirements.txt, -r requirements/test.txt, importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/requirements.in

@@ -3,6 +3,7 @@ msgpack~=0.6.1
 Rtree~=0.9.4
 certifi
 Pygments
+jsonschema
 
 # these are only here for dependencies
 psycopg2==2.8.5

requirements/test.txt

@@ -6,38 +6,40 @@
 #
 aspectlib==1.5.1          # via pytest-benchmark
 atomicwrites==1.4.0       # via pytest
-attrs==19.3.0             # via pytest
+attrs==19.3.0             # via -r requirements/../requirements.txt, jsonschema, pytest
 cached-property==1.5.1    # via -r requirements/../requirements.txt
 certifi==2020.6.20        # via -r requirements/../requirements.txt
 cffi==1.14.0              # via -r requirements/../requirements.txt
 click==7.1.2              # via -r requirements/../requirements.txt
 coverage==5.1             # via pytest-cov
 fields==5.0.0             # via aspectlib
 gprof2dot==2019.11.30     # via pytest-profiling
-html5lib==1.0.1           # via -r requirements/test.in
-importlib-metadata==1.6.1  # via pluggy, pytest
+html5lib==1.1             # via -r requirements/test.in
+importlib-metadata==1.6.1  # via -r requirements/../requirements.txt, jsonschema, pluggy, pytest
+jsonschema==3.2.0         # via -r requirements/../requirements.txt
 lovely-pytest-docker==0.1.0  # via -r requirements/test.in
 more-itertools==8.4.0     # via pytest
 msgpack==0.6.2            # via -r requirements/../requirements.txt
 packaging==20.4           # via pytest, pytest-sugar
 pluggy==0.13.1            # via pytest
 py-cpuinfo==6.0.0         # via pytest-benchmark
-py==1.8.2                 # via pytest
+py==1.9.0                 # via pytest
 pycparser==2.20           # via -r requirements/../requirements.txt, cffi
 pygments==2.6.1           # via -r requirements/../requirements.txt
 pyparsing==2.4.7          # via packaging
+pyrsistent==0.16.0        # via -r requirements/../requirements.txt, jsonschema
 pytest-benchmark[aspect]==3.2.3  # via -r requirements/test.in
 pytest-cov==2.10.0        # via -r requirements/test.in
 pytest-helpers-namespace==2019.1.8  # via -r requirements/test.in
 pytest-profiling==1.7.0   # via -r requirements/test.in
 pytest-sugar==0.9.3       # via -r requirements/test.in
 pytest==4.6.11            # via -r requirements/test.in, lovely-pytest-docker, pytest-benchmark, pytest-cov, pytest-helpers-namespace, pytest-profiling, pytest-sugar
 rtree==0.9.4              # via -r requirements/../requirements.txt
-six==1.15.0               # via html5lib, packaging, pytest, pytest-profiling
+six==1.15.0               # via -r requirements/../requirements.txt, html5lib, jsonschema, packaging, pyrsistent, pytest, pytest-profiling
 termcolor==1.1.0          # via pytest-sugar
-wcwidth==0.2.4            # via pytest
+wcwidth==0.2.5            # via pytest
 webencodings==0.5.1       # via html5lib
-zipp==3.1.0               # via importlib-metadata
+zipp==3.1.0               # via -r requirements/../requirements.txt, importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

sno/cli_util.py

@@ -1,4 +1,6 @@
+import json
 import click
+import jsonschema
 
 
 class MutexOption(click.Option):
@@ -74,6 +76,29 @@ def convert(self, value, param, ctx):
         return value
 
 
+class JsonFromFile(StringFromFile):
+    name = 'json'
+
+    def __init__(self, schema=None, **file_kwargs):
+        super().__init__(**file_kwargs)
+        self.schema = schema
+
+    def convert(self, value, param, ctx):
+        value = super().convert(value, param, ctx)
+        try:
+            value = json.loads(value)
+        except json.JSONDecodeError as e:
+            self.fail(
+                f"Invalid JSON: {e}", param, ctx,
+            )
+        if self.schema:
+            try:
+                jsonschema.validate(instance=value, schema=self.schema)
+            except jsonschema.ValidationError as e:
+                self.fail(str(e), param, ctx)
+        return value
+
+
 def call_and_exit_flag(*args, callback, **kwargs):
     """
     Add an is_flag option that, when set, eagerly calls the given callback with only the context as a parameter.

sno/gpkg.py

@@ -71,7 +71,7 @@ def db(path, **kwargs):
     return db
 
 
-def get_meta_info(db, layer):
+def get_meta_info(db, layer, exclude_keys=()):
     """
     Returns metadata from the gpkg_* tables about this GPKG.
     Keep this in sync with OgrImporter.build_meta_info for other datasource types.
@@ -132,12 +132,14 @@ def get_meta_info(db, layer):
         ),
     }
     try:
-        for filename, (sql, params, rtype) in QUERIES.items():
+        for key, (sql, params, rtype) in QUERIES.items():
+            if key in exclude_keys:
+                continue
             # check table exists, the metadata ones may not
-            if not filename.startswith("sqlite_"):
+            if not key.startswith("sqlite_"):
                 dbcur.execute(
                     "SELECT name FROM sqlite_master WHERE type='table' AND name=?;",
-                    (filename,),
+                    (key,),
                 )
                 if not dbcur.fetchone():
                     continue
@@ -148,9 +150,9 @@ def get_meta_info(db, layer):
             ]
             if rtype is dict:
                 value = value[0] if len(value) else None
-            yield (filename, value)
+            yield (key, value)
     except Exception:
-        print(f"Error building meta/{filename}")
+        print(f"Error building meta/{key}")
         raise