91 asdf external file (#215)

* Add possibility to document and serialize external files in an asdf file

.typo-ci.yml

@@ -71,6 +71,9 @@ excluded_words:
   - timedeltaindex
   - datetimeindex
   - isoformat
+  # pyfilesystem --------------------------------
+  - osfs
+  - memoryfs
   # pytest --------------------------------------
   - conftest
   - addoption

CHANGELOG.md

@@ -2,9 +2,12 @@
 
 ## 0.2.3 (unreleased)
 ### added
-- add `weldx.transformations.CoordinateSystemManager.relabel` function [[#219]](https://github.com/BAMWelDX/weldx/pull/212)
+- add `weldx.transformations.CoordinateSystemManager.relabel` function [[#219]](https://github.com/BAMWelDX/weldx/pull/219)
 
 ### ASDF
+- Add possibility to store meta data and content of an external file in an ASDF file [[#215]](https://github.com/BAMWelDX/weldx/pull/215)
+  - Python class: `weldx.asdf.ExternalFile`
+  - Schema: `core/file-1.0.0.yaml`
 - Added support for serializing generic metadata and userdata attributes for weldx classes. [[#209]](https://github.com/BAMWelDX/weldx/pull/209)
   - the provisional attribute names are `wx_metadata` and `wx_user`
 - `None` values are removed from the asdf tree for all `weldx` classes. [[#212]](https://github.com/BAMWelDX/weldx/pull/212)
@@ -22,6 +25,10 @@
 - fix "datetime64" passing for "timedelta64" in `xr_check_coords` [[#221]](https://github.com/BAMWelDX/weldx/pull/221)
 - fix `time_ref_restore` not working correctly if no `time_ref` was set [[#221]](https://github.com/BAMWelDX/weldx/pull/221)
 
+### dependencies
+- Add [PyFilesystem](https://docs.pyfilesystem.org/en/latest/)(`fs`) as new dependency 
+
+
 ## 0.2.2 (30.11.2020)
 ### added
 - Added `weldx.utility.ureg_check_class` class decorator to enable `pint` dimensionality checks with `@dataclass`. [[#179]](https://github.com/BAMWelDX/weldx/pull/179)

doc/schemas/core.rst

@@ -11,6 +11,7 @@ The ``core`` directory contains schema implementations for the base classes.
     core/dimension-1.0.0
     core/data_array-1.0.0
     core/dataset-1.0.0
+    core/file-1.0.0
 
 Transformations
 ===============

environment.yml

@@ -15,6 +15,7 @@ dependencies:
   - boltons
   - asdf>=2.7
   - openpyxl
+  - fs
   # graph packages
   - networkx
   # Code quality

setup.cfg

@@ -49,6 +49,7 @@ install_requires =
     boltons
     networkx >=2
     matplotlib >=3
+    fs
     ipywidgets
 include_package_data = True
 

tests/_helpers.py

@@ -1,5 +1,7 @@
 """Provides some utility functions for tests."""
 
+from typing import Any
+
 import numpy as np
 
 import weldx.transformations as tf
@@ -66,3 +68,22 @@ def are_all_columns_unique(matrix, decimals=3):
     """
     unique = np.unique(np.round(matrix, decimals=decimals), axis=1)
     return unique.shape[0] == matrix.shape[0] and unique.shape[1] == matrix.shape[1]
+
+
+def get_test_name(param: Any) -> str:
+    """Get the test name from the parameter list of a parametrized test.
+
+    Parameters
+    ----------
+    param : Any
+        A parameter of the test
+
+    Returns
+    -------
+    str :
+        The name of the test or an empty string.
+
+    """
+    if isinstance(param, str) and param[0] == "#":
+        return param[1:]
+    return ""

tests/asdf_tests/test_asdf_core.py

@@ -1,12 +1,19 @@
 """Tests asdf implementations of core module."""
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
 import numpy as np
 import pandas as pd
 import pytest
 import xarray as xr
 from asdf import ValidationError
+from fs.memoryfs import MemoryFS
+from fs.osfs import OSFS
 from scipy.spatial.transform import Rotation
 
 import weldx.transformations as tf
+from tests._helpers import get_test_name
+from weldx.asdf.tags.weldx.core.file import ExternalFile
 from weldx.asdf.utils import _write_buffer, _write_read_buffer
 from weldx.constants import WELDX_QUANTITY as Q_
 from weldx.core import MathematicalExpression as ME  # nopep8
@@ -396,3 +403,205 @@ def test_time_series_discrete(ts, copy_arrays, lazy_load):
         assert np.all(ts_file.data == ts.data)
     assert np.all(ts_file.time == ts.time)
     assert ts_file.interpolation == ts.interpolation
+
+
+# --------------------------------------------------------------------------------------
+# ExternalFile
+# --------------------------------------------------------------------------------------
+
+weldx_root_dir = Path(__file__).parent.parent.parent.absolute().as_posix()
+
+
+class TestExternalFile:
+    """Collects all tests related to the `ExternalFile` class."""
+
+    # test_init ------------------------------------------------------------------------
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "file_path, save_content, hostname",
+        [
+            (f"{weldx_root_dir}/doc/_static/WelDX_notext.ico", True, "a host"),
+            (f"{weldx_root_dir}/doc/_static/WelDX_notext.ico", False, "a host"),
+            (Path(f"{weldx_root_dir}/doc/_static/WelDX_notext.ico"), False, "a host"),
+            (f"{weldx_root_dir}/doc/_static/WelDX_notext.ico", False, None),
+        ],
+    )
+    def test_init(file_path, save_content, hostname):
+        """Test the `__init__` method.
+
+        Parameters
+        ----------
+        file_path: Union[str, Path]
+            Path of the file
+        save_content : bool
+            If `True`, the file should be stored in the asdf file
+        hostname:
+            The files hostname
+
+        """
+        ef = ExternalFile(file_path, asdf_save_content=save_content, hostname=hostname)
+        assert save_content == ef.asdf_save_content
+        assert ef.filename == "WelDX_notext.ico"
+        assert ef.suffix == "ico"
+
+        if hostname is not None:
+            assert hostname == ef.hostname
+        else:
+            print(hostname)
+            assert isinstance(ef.hostname, str)
+
+    # test_init_exceptions -------------------------------------------------------------
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "kwargs, exception_type, test_name",
+        [
+            ({"path": "does_not.exist"}, ValueError, "# File does not exist"),
+            ({"hashing_algorithm": "fancy"}, ValueError, "# Invalid hashing algorithm"),
+        ],
+        ids=get_test_name,
+    )
+    def test_init_exceptions(kwargs, exception_type, test_name):
+        """Test the `__init__` methods exceptions.
+
+        Parameters
+        ----------
+        kwargs : Dict
+            Key word arguments that should be passed to the `__init__` method
+        exception_type :
+            The expected exception type
+        test_name : str
+            Name of the test
+
+        """
+        if "path" not in kwargs:
+            kwargs["path"] = f"{weldx_root_dir}/doc/_static/WelDX_notext.ico"
+
+        with pytest.raises(exception_type):
+            ExternalFile(**kwargs)
+
+    # test_write_to --------------------------------------------------------------------
+    @staticmethod
+    @pytest.mark.parametrize(
+        "dir_read, file_name",
+        [
+            ("doc/_static", "WelDX_notext.ico"),
+            ("doc/_static", "WelDX_notext.svg"),
+            ("weldx", "transformations.py"),
+        ],
+    )
+    def test_write_to(dir_read, file_name):
+        """Test the `write_to` method by writing a read file to a virtual file system.
+
+        Parameters
+        ----------
+        dir_read : str
+            Directory that contains the source file
+        file_name : str
+            Name of the source file
+
+        """
+        path_read = f"{dir_read}/{file_name}"
+        ef = ExternalFile(f"{weldx_root_dir}/{path_read}")
+
+        with OSFS(weldx_root_dir) as file_system:
+            original_hash = file_system.hash(path_read, "md5")
+
+            # check writing to hard drive
+            with TemporaryDirectory(dir=weldx_root_dir) as td:
+                ef.write_to(td)
+                new_file_path = Path(f"{Path(td).name}/{file_name}").as_posix()
+                assert file_system.isfile(new_file_path)
+                assert file_system.hash(new_file_path, "md5") == original_hash
+
+        # check writing to a memory file system
+        with MemoryFS() as file_system:
+            file_system.makedir("some_directory")
+            ef.write_to("some_directory", file_system)
+
+            new_file_path = f"some_directory/{file_name}"
+            assert file_system.isfile(new_file_path)
+            assert file_system.hash(new_file_path, "md5") == original_hash
+
+    # test_hashing ---------------------------------------------------------------------
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "algorithm, buffer_size",
+        [
+            ("SHA-256", 1024),
+            ("MD5", 2048),
+        ],
+    )
+    def test_hashing(algorithm: str, buffer_size: int):
+        """Test the hashing functions.
+
+        Two things should be tested here:
+        - All available algorithms can be selected and work properly
+        - Both available hashing functions deliver the same hash for the same algorithm
+
+        Parameters
+        ----------
+        algorithm : str
+            The hashing algorithm
+        buffer_size : int
+            The size of the buffer that is used by the `calculate_hash` method.
+
+        """
+        file_path = f"{weldx_root_dir}/doc/_static/WelDX_notext.ico"
+        ef = ExternalFile(file_path, hashing_algorithm=algorithm)
+        buffer = ef.get_file_content()
+
+        hash_buffer = ExternalFile.calculate_hash(buffer, algorithm)
+        hash_file = ExternalFile.calculate_hash(file_path, algorithm, buffer_size)
+
+        assert hash_buffer == hash_file
+
+    # test_asdf_serialization ----------------------------------------------------------
+
+    @staticmethod
+    @pytest.mark.parametrize("copy_arrays", [True, False])
+    @pytest.mark.parametrize("lazy_load", [True, False])
+    @pytest.mark.parametrize("store_content", [True, False])
+    def test_asdf_serialization(copy_arrays, lazy_load, store_content):
+        """Test the asdf serialization of the `ExternalFile` class.
+
+        Parameters
+        ----------
+        copy_arrays : bool
+            If `False`, arrays are accessed via memory mapping whenever possible while
+            reading them.
+        lazy_load : bool
+            If `True`, items from the asdf file are not loaded until accessed.
+        store_content : bool
+            If `True`, the file content is stored in the asdf file.
+
+        """
+        ef = ExternalFile(
+            f"{weldx_root_dir}/doc/_static/WelDX_notext.ico",
+            asdf_save_content=store_content,
+        )
+        tree = {"file": ef}
+        ef_file = _write_read_buffer(
+            tree, open_kwargs={"copy_arrays": copy_arrays, "lazy_load": lazy_load}
+        )["file"]
+
+        assert ef.filename == ef_file.filename
+        assert ef.suffix == ef_file.suffix
+        assert ef.directory == ef_file.directory
+        assert ef.hostname == ef_file.hostname
+
+        assert ef.created == ef_file.created
+        assert ef.modified == ef_file.modified
+        assert ef.size == ef_file.size
+
+        assert ef.hashing_algorithm == ef_file.hashing_algorithm
+
+        if store_content:
+            with OSFS(weldx_root_dir) as file_system:
+                original_hash = file_system.hash("doc/_static/WelDX_notext.ico", "md5")
+
+            with MemoryFS() as file_system:
+                ef_file.write_to("", file_system)
+                assert file_system.hash("WelDX_notext.ico", "md5") == original_hash

tests/test_core.py

@@ -7,19 +7,11 @@
 import pytest
 
 import weldx.utility as ut
+from tests._helpers import get_test_name
 from weldx.constants import WELDX_QUANTITY as Q_
 from weldx.constants import WELDX_UNIT_REGISTRY as UREG
 from weldx.core import MathematicalExpression, TimeSeries
 
-
-# Todo: Move this to conftest.py?
-def get_test_name(param):
-    """Get the test name from the parameter list of a parametrized test."""
-    if isinstance(param, str) and param[0] == "#":
-        return param[1:]
-    return ""
-
-
 # --------------------------------------------------------------------------------------
 # MathematicalExpression
 # --------------------------------------------------------------------------------------

tests/test_transformations.py

@@ -15,20 +15,13 @@
 
 import weldx.transformations as tf
 import weldx.utility as ut
+from tests._helpers import get_test_name
 from weldx import Q_
 from weldx.transformations import LocalCoordinateSystem as LCS  # noqa
 
 # helpers for tests -----------------------------------------------------------
 
 
-# Todo: Move this to conftest.py?
-def get_test_name(param):
-    """Get the test name from the parameter list of a parametrized test."""
-    if isinstance(param, str) and param[0] == "#":
-        return param[1:]
-    return ""
-
-
 def check_matrix_does_not_reflect(matrix):
     """Check if a matrix does not reflect.
 

weldx/asdf/__init__.py

@@ -1,3 +1,9 @@
+"""
+isort:skip_file
+"""
 from weldx.asdf import tags  # implement tags before the asdf extensions
 from weldx.asdf import constants, utils
 from weldx.asdf.extension import WeldxAsdfExtension, WeldxExtension
+
+# class imports to weldx.asdf namespace
+from weldx.asdf.tags.weldx.core.file import ExternalFile