Merge branch 'master' into 289_csm_interp_time_range

.deepsource.toml

@@ -10,6 +10,7 @@ exclude_patterns = [
   "scripts/**",
   "weldx/_version.py",
   "weldx/asdf/**",
+  "weldx/tags/**",
 ]
 
 [[analyzers]]

.github/workflows/pytest_asdf.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-latest]
         py: ['3.8']
     steps:
       - uses: actions/checkout@v2
@@ -19,4 +19,28 @@ jobs:
           pip install -e .[test]
       - name: run asdf schema pytest
         run: |
-          pytest --asdf-tests --ignore=weldx/tests/ --no-cov weldx/asdf/schemas/
+          pytest --asdf-tests --ignore=weldx/tests/ --no-cov weldx/schemas/
+
+  validate_manifest:
+    name: validate weldx manifest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: pip installs
+        run: |
+          python -m pip install git+https://github.com/asdf-format/asdf.git
+      - uses: jannekem/run-python-script-action@v1
+        with:
+          script: |
+            from pathlib import Path
+            import asdf
+            import yaml
+            schema = asdf.schema.load_schema(
+                "asdf://asdf-format.org/core/schemas/extension_manifest-1.0.0"
+            )
+            for f in Path("./weldx/manifests/").glob("*.yaml"):
+                manifest = yaml.safe_load(open(f).read())
+                asdf.schema.validate(manifest, schema=schema)

CHANGELOG.md

@@ -39,7 +39,7 @@
 ### fixes
 
 - `WeldxFile.show_asdf_header` prints output on console, before it only returned the header as parsed dict and string
-  representation. Also tweaked efficency by not writing binary
+  representation. Also tweaked efficiency by not writing binary
   blocks [[#459]](https://github.com/BAMWelDX/weldx/pull/459).
 
 ### documentation
@@ -50,6 +50,21 @@
 ### ASDF
 
 - add ``time/time`` schema to support `Time` class [[#463]](https://github.com/BAMWelDX/weldx/pull/463).
+- rework ASDF extension to new asdf 2.8 API [[#467]](https://github.com/BAMWelDX/weldx/pull/467)
+    - move schema files to ``weldx/schemas``
+    - create extension manifest in ``weldx/manifests``. The manifest also contains tag mappings for legacy tag names for
+      backwards compatibility.
+    - move tag module to ``weldx/tags``
+    - refactor all asdf uris to new ``asdf://`` naming convention,
+      see https://asdf.readthedocs.io/en/latest/asdf/extending/uris.html#entities-identified-by-uri
+    - replaced all referenced weldx tag versions in schemas with ``1.*``
+    - refactor ``asdf://weldx.bam.de/weldx/schemas/datamodels/single_pass_weld-1.0.0.schema``
+      to ``asdf://weldx.bam.de/weldx/schemas/datamodels/single_pass_weld-1.0.0`` and enable schema test
+    - add legacy class for validators support in ``weldx.asdf._extension.py``
+    - asdf utility functions `weldx.asdf.util.uri_match`, `weldx.asdf.util.get_converter_for_tag`
+      and `weldx.asdf.util.get_weldx_extension`
+    - add ``devtools/scripts/update_manifest.py`` to auto update manifest from extension metadata
+    - custom shape validation must now be implemented via staticmethod ``WeldxConverter.shape_from_tagged``
 
 ### deprecations
 

MANIFEST.in

@@ -1,4 +1,5 @@
-recursive-include weldx/asdf/schemas *.yaml
+recursive-include weldx/schemas *.yaml
+recursive-include weldx/manifests *.yaml
 
 prune .binder
 prune .github

devtools/conda.recipe/meta.yaml

@@ -10,7 +10,7 @@ source:
 build:
   number: 0
   noarch: python
-  script: pip install . -v
+  script: {{ PYTHON }} -m pip install . -vv
   entry_points:
   {% for e in data['entry_points']['console_scripts'] %}
     - {{ e }}
@@ -37,6 +37,8 @@ test:
     - weldx
     - weldx.visualization
   commands:
+    - pip check
+    - welding_schema --help
     - pytest --pyargs weldx.tests -n auto --dist=loadfile
 
 about:

devtools/scripts/update_manifest.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+
+import yaml
+
+from weldx.asdf.types import _legacy_tag_from_new_tag
+from weldx.asdf.util import get_converter_for_tag
+
+
+def update_manifest(
+    search_dir: str = "../../schemas",
+    out: str = "../../manifests/weldx-1.0.0.yaml",
+):
+    """Create manifest file from existing schemas."""
+    # read existing manifest
+    manifest = yaml.load(
+        Path(out).read_text(),
+        Loader=yaml.SafeLoader,
+    )
+
+    # keep only ASDF schema mappings
+    manifest["tags"] = [
+        mapping
+        for mapping in manifest["tags"]
+        if mapping["schema_uri"].startswith("http://stsci.edu/schemas")
+    ]
+
+    schemas = Path(search_dir).rglob("*.yaml")
+
+    for schema in schemas:
+        content = yaml.load(
+            schema.read_text(),
+            Loader=yaml.SafeLoader,
+        )
+        if "id" in content:  # should be schema file
+            uri: str = content["id"]
+            tag = uri.replace("/schemas/", "/tags/")
+            if get_converter_for_tag(tag):  # check if converter is implemented
+                manifest["tags"].append(dict(tag_uri=tag, schema_uri=uri))
+                manifest["tags"].append(
+                    dict(tag_uri=_legacy_tag_from_new_tag(tag), schema_uri=uri)
+                )  # legacy_tag
+            else:
+                print(f"No converter for URI: {schema}")
+
+    with open(Path(out), "w") as outfile:
+        outfile.write("%YAML 1.1\n---\n")
+        yaml.dump(
+            manifest,
+            outfile,
+            default_flow_style=False,
+            sort_keys=False,
+        )
+        outfile.write("...\n")
+
+
+if __name__ == "__main__":
+    update_manifest()

doc/conf.py

@@ -53,7 +53,7 @@ def _prevent_sphinx_circular_imports_bug():
     raise
 
 import weldx.visualization  # load visualization (currently no auto-import in pkg).
-from weldx.asdf.constants import SCHEMA_PATH, WELDX_TAG_BASE
+from weldx.asdf.constants import SCHEMA_PATH, WELDX_TAG_URI_BASE
 
 # -- copy tutorial files to doc folder -------------------------------------------------
 tutorials_dir = pathlib.Path("./tutorials")
@@ -196,10 +196,10 @@ def _prevent_sphinx_circular_imports_bug():
 # -- sphinx-asdf configuration -------------------------------------------------
 # This variable indicates the top-level directory containing schemas.
 # The path is relative to the location of conf.py in the package
-asdf_schema_path = os.path.relpath(SCHEMA_PATH)
+asdf_schema_path = os.path.relpath(str(SCHEMA_PATH))
 # This variable indicates the standard prefix that is common to all schemas
 # provided by the package.
-asdf_schema_standard_prefix = "weldx.bam.de/weldx"
+asdf_schema_standard_prefix = ""  # SCHEMA_PATH already points to final schema dir
 
 # enable references to the ASDF Standard documentation
 asdf_schema_reference_mappings = [
@@ -208,7 +208,7 @@ def _prevent_sphinx_circular_imports_bug():
         "http://asdf-standard.readthedocs.io/en/latest/generated/stsci.edu/asdf/",
     ),
     (
-        WELDX_TAG_BASE,
+        WELDX_TAG_URI_BASE,
         "http://weldx.readthedocs.io/en/latest/generated/weldx.bam.de/weldx/",
     ),
 ]
@@ -274,7 +274,7 @@ def _prevent_sphinx_circular_imports_bug():
     # "numba": ("https://numba.pydata.org/numba-doc/latest", None),
     "pint": ("https://pint.readthedocs.io/en/stable", None),
     "jsonschema": ("https://python-jsonschema.readthedocs.io/en/stable/", None),
-    "asdf": ("https://asdf.readthedocs.io/en/stable/", None),
+    "asdf": ("https://asdf.readthedocs.io/en/latest/", None),
     "networkx": ("https://networkx.org/documentation/stable/", None),
     "IPython": ("https://ipython.readthedocs.io/en/stable/", None),
     "k3d": ("https://k3d-jupyter.org/", None),

doc/nitpick_ignore

@@ -52,3 +52,8 @@ py:obj left
 py:obj right
 py:meth scipy.stats.special_ortho_group
 py:obj mrp[i]
+
+# asdf
+py:class iterable of asdf.extension.Compressor instances
+py:class See the class docstring for details on keyword
+py:class iterable of str

doc/standard/shape-validation.rst

@@ -223,7 +223,7 @@ The following custom types can be validate with ``wx_shape`` even though
 the might not always define a shape property in itself.
 
 - ``number`` will validate like ``shape: [1]``
-- ``tag:weldx.bam.de:weldx/time/timedeltaindex-1.0.0`` will validate
+- ``asdf://weldx.bam.de/weldx/tags/time/timedeltaindex-1.0.0`` will validate
   against the length of the ``TimedeltaIndex`` even if no data is
   stored.
 
@@ -237,8 +237,8 @@ Here is a more complex example demonstration some of the above points.
    %YAML 1.1
    ---
    $schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
-   id: "http://weldx.bam.de/schemas/weldx/debug/test_shape_validator-1.0.0"
-   tag: "tag:weldx.bam.de:weldx/debug/test_shape_validator-1.0.0"
+   id: "asdf://weldx.bam.de/weldx/schemas/debug/test_shape_validator-1.0.0"
+   tag: "asdf://weldx.bam.de/weldx/tags/debug/test_shape_validator-1.0.0"
 
    title: |
      simple demonstration and test schema for wx_shape validator syntax

doc/standard/unit-validation.rst

@@ -48,7 +48,7 @@ this more complex example for details:
 
 .. code:: yaml
 
-   tag: "tag:weldx.bam.de:weldx/debug/test_unit_validator-1.0.0"
+   tag: "asdf://weldx.bam.de/weldx/tags/debug/test_unit_validator-1.0.0"
 
    title: |
      simple demonstration schema for wx_unit validator

doc/standard/welding-processes.rst

@@ -10,7 +10,7 @@ base GMAW process schema
 ------------------------
 
 The main layout of any GMAW process is defined in the
-``http://weldx.bam.de/schemas/weldx/process/terms-1.0.0`` schema as
+``asdf://weldx.bam.de/weldx/schemas/process/terms-1.0.0`` schema as
 ``base_process``:
 
 .. code:: yaml
@@ -29,7 +29,7 @@ The main layout of any GMAW process is defined in the
          type: string
        parameters:
          type: object
-         wx_property_tag: "tag:weldx.bam.de:weldx/core/time_series-*"
+         wx_property_tag: "asdf://weldx.bam.de/weldx/tags/core/time_series-*"
        meta:
          type: object
      required: [base_process,manufacturer,power_source,parameters]
@@ -55,15 +55,15 @@ simple generic GMAW process definition
 --------------------------------------
 
 The most generic tag implementation of any arc welding process is
-provided by ``tag:weldx.bam.de:weldx/process/GMAW-1.0.0``:
+provided by ``asdf://weldx.bam.de/weldx/tags/process/GMAW-1.0.0``:
 
 .. code:: yaml
 
    %YAML 1.1
    ---
    $schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
-   id: "http://weldx.bam.de/schemas/weldx/process/GMAW-1.0.0"
-   tag: "tag:weldx.bam.de:weldx/process/GMAW-1.0.0"
+   id: "asdf://weldx.bam.de/weldx/schemas/process/GMAW-1.0.0"
+   tag: "asdf://weldx.bam.de/weldx/tags/process/GMAW-1.0.0"
 
    title: |
      Generic GMAW process definition.
@@ -117,13 +117,13 @@ that the ``parameters`` property always includes a ``wire_feedrate`` and
      wire_feedrate:
        description: |
          Nominal average wire feedrate.
-       tag: "tag:weldx.bam.de:weldx/core/time_series-1.0.0"
+       tag: "asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0"
        wx_unit: "m/s"
 
      voltage:
        description: |
          Nominal target voltage for spray arc processes.
-       tag: "tag:weldx.bam.de:weldx/core/time_series-1.0.0"
+       tag: "asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0"
        wx_unit: "V"
 
 combining process schemas
@@ -167,8 +167,8 @@ parameters:
    %YAML 1.1
    ---
    $schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
-   id: "http://weldx.bam.de/schemas/weldx/process/CLOOS/spray_arc-1.0.0"
-   tag: "tag:weldx.bam.de:weldx/process/CLOOS/spray_arc-1.0.0"
+   id: "asdf://weldx.bam.de/weldx/schemas/process/CLOOS/spray_arc-1.0.0"
+   tag: "asdf://weldx.bam.de/weldx/tags/process/CLOOS/spray_arc-1.0.0"
 
    title: |
      CLOOS spray arc process.
@@ -182,10 +182,10 @@ parameters:
            type: object
            properties:
              impedance:
-               tag: "tag:weldx.bam.de:weldx/core/time_series-1.0.0"
+               tag: "asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0"
                wx_unit: "percent"
              characteristic:
-               tag: "tag:weldx.bam.de:weldx/core/time_series-1.0.0"
+               tag: "asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0"
                wx_unit: "V/A"
            required: [impedance, characteristic]
 
@@ -221,20 +221,20 @@ And here is the resulting ASDF snippet:
 
 .. code:: yaml
 
-   spray: !<tag:weldx.bam.de:weldx/process/CLOOS/spray_arc-1.0.0>
+   spray: !<asdf://weldx.bam.de/weldx/tags/process/CLOOS/spray_arc-1.0.0>
      base_process: spray
      manufacturer: CLOOS
      parameters:
-       characteristic: !<tag:weldx.bam.de:weldx/core/time_series-1.0.0>
+       characteristic: !<asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0>
          unit: volt / ampere
          values: 5
-       impedance: !<tag:weldx.bam.de:weldx/core/time_series-1.0.0>
+       impedance: !<asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0>
          unit: percent
          values: 10.0
-       voltage: !<tag:weldx.bam.de:weldx/core/time_series-1.0.0>
+       voltage: !<asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0>
          unit: volt
          values: 40.0
-       wire_feedrate: !<tag:weldx.bam.de:weldx/core/time_series-1.0.0>
+       wire_feedrate: !<asdf://weldx.bam.de/weldx/tags/core/time_series-1.0.0>
          unit: meter / minute
          values: 10.0
      power_source: Quinto