resolve problems with unit test workflow

.github/workflows/conda_unit_test.yml

@@ -21,13 +21,13 @@ jobs:
     runs-on: ubuntu-latest
     # needs: lint
     strategy:
+      max-parallel: 5
       matrix:
         python-version:
         - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"
-      max-parallel: 5
 
     steps:
     - uses: actions/checkout@v4
@@ -71,6 +71,6 @@ jobs:
     - name: Test with coverage and pytest
       shell: bash -l {0}
       run: |
-        coverage run --concurrency=thread --parallel-mode -m pytest -vvv
+        coverage run --concurrency=thread --parallel-mode -m pytest -vvv  # --exitfirst .
         coverage combine
         coverage report

.github/workflows/publish-docs.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: [3.11]
 
     steps:
     - name: Set env vars

RELEASE_NOTES.rst

@@ -39,6 +39,9 @@ Fixes
 -----
 
 * ``diffract.forward()`` should pick solution consistent with ``diffract.forward_solution_table()``, if it can. Otherwise, fall back to previous iterative method.
+* Make ``util.list_orientation_runs()`` work with databroker v1.2 or v2+.
+* Make ``util.run_orientation_info()`` work with databroker v1.2 or v2+.
+* Resolved under-reported problems in CI unit tests.
 * ``util.restore_reflections()`` use renamed motor axes if so defined.
 
 Maintenance

docs/source/configuration.rst

@@ -169,7 +169,67 @@ API
 .. automodule:: hkl.configuration
     :members:
         DiffractometerConfiguration,
-        DCConstraint, DCLattice, DCReflection, DCSample, DCConfiguration,
         _check_key, _check_not_value, _check_range, _check_type, _check_value
     :private-members:
     :undoc-members:
+
+.. autoclass:: hkl.configuration.DCConfiguration
+
+    .. autoattribute:: geometry
+    .. autoattribute:: engine
+    .. autoattribute:: library
+    .. autoattribute:: mode
+    .. autoattribute:: canonical_axes
+    .. autoattribute:: real_axes
+    .. autoattribute:: reciprocal_axes
+    .. autoattribute:: samples
+    .. autoattribute:: samples
+    .. autoattribute:: name
+    .. autoattribute:: datetime
+    .. autoattribute:: wavelength_angstrom
+    .. autoattribute:: energy_keV
+    .. autoattribute:: hklpy_version
+    .. autoattribute:: library_version
+    .. autoattribute:: python_class
+    .. autoattribute:: other
+    .. automethod:: validate
+    .. automethod:: write
+
+.. autoclass:: hkl.configuration.DCConstraint
+
+    .. autoattribute:: low_limit
+    .. autoattribute:: high_limit
+    .. autoattribute:: value
+    .. autoattribute:: fit
+    .. automethod:: validate
+    .. autoproperty:: values
+
+.. autoclass:: hkl.configuration.DCLattice
+
+    .. autoattribute:: a
+    .. autoattribute:: b
+    .. autoattribute:: c
+    .. autoattribute:: alpha
+    .. autoattribute:: beta
+    .. autoattribute:: gamma
+    .. automethod:: validate
+    .. autoproperty:: values
+
+.. autoclass:: hkl.configuration.DCReflection
+
+    .. autoattribute:: reflection
+    .. autoattribute:: position
+    .. autoattribute:: wavelength
+    .. autoattribute:: orientation_reflection
+    .. autoattribute:: flag
+    .. automethod:: validate
+
+.. autoclass:: hkl.configuration.DCSample
+
+    .. autoattribute:: name
+    .. autoattribute:: lattice
+    .. autoattribute:: reflections
+    .. autoattribute:: UB
+    .. autoattribute:: U
+    .. automethod:: validate
+    .. automethod:: write

env-dev.yml

@@ -16,6 +16,7 @@ dependencies:
   - apischema
   - black
   - bluesky
+  - chardet
   - databroker
   - flake8
   - hkl
@@ -35,6 +36,7 @@ dependencies:
   - spec2nexus
   - sphinx
   - tqdm
+  - trio >=0.22
   - pip:
     - coveralls
     - pytest

environment.yml

@@ -15,6 +15,7 @@ dependencies:
   - python >=3.8,<3.12
   - apischema
   - bluesky
+  - databroker
   - hkl
   - numpy
   - ophyd
@@ -25,6 +26,10 @@ dependencies:
   - pygobject
   - pyRestTable
   - spec2nexus
+  - suitcase-jsonl
+  - suitcase-mongo
+  - suitcase-msgpack
+  - suitcase-utils
   - tqdm
   - pip:
     - coveralls

hkl/calc.py

@@ -800,6 +800,11 @@ def __repr__(self):
     def __str__(self):
         return repr(self)
 
+    @property
+    def _cfg_reciprocal(self):
+        """Return reciprocal lattice to save as configuration."""
+        return tuple(list(self.sample.reciprocal))
+
 
 class CalcE4CH(CalcRecip):
     """Geometry: E4CH"""

hkl/configuration.py

@@ -38,6 +38,8 @@
 from dataclasses import asdict
 from dataclasses import dataclass
 from dataclasses import field
+from typing import Dict
+from typing import List
 
 import numpy
 import pyRestTable
@@ -187,13 +189,13 @@ def values(self):
 class DCReflection:
     """(internal) Configuration of one orientation reflection."""
 
-    reflection: dict[str, float]
+    reflection: Dict[str, float]
     """
     Reciprocal-space axis positions.  Keys must match in the list of
     ``reciprocal_axes``.
     """
 
-    position: dict[str, float]
+    position: Dict[str, float]
     """
     Real-space axis positions.  Keys must match in the list of
     ``canonical_axes``.
@@ -233,17 +235,18 @@ class DCSample:
     lattice: DCLattice
     """Crystal lattice parameters (angstroms and degrees)"""
 
-    reflections: list[DCReflection]
+    reflections: List[DCReflection]
     """List of orientation reflections."""
 
-    UB: list[list[float]]
+    UB: List[List[float]]
     """
     Orientation matrix (3 x 3).  U is the crystal orientation matrix relative
     to the diffractometer and B is the transition matrix of a non-orthonormal
     (the reciprocal of the crystal) in an orthonormal system.
     """
 
-    U: list[list[float]] = field(default_factory=list[list[float]])
+    # TODO: Once py38 is dropped, re-enable the default value setting
+    U: List[List[float]]  # = field(default_factory=list[list[float]])
     """
     Orientation matrix (3 x 3) of the crystal relative to the diffractometer.
     (optional)
@@ -337,36 +340,36 @@ class DCConfiguration:
     diffractometer to restore.
     """
 
-    canonical_axes: list[str]
+    canonical_axes: List[str]
     """
     List of the diffractometer real-space axis names.  Both the exact spelling
     and order are defined by the back-end computation library.  MUST match
     diffractometer to restore.
     """
 
-    real_axes: list[str]
+    real_axes: List[str]
     """
     User-defined real-space axis names. MUST match diffractometer to restore.
     The length and order of this list must be the same as the
     ``canonical_axes``. It is used to resolve any (real-space) ``positioner``
     names in this file.
     """
 
-    reciprocal_axes: list[str]
+    reciprocal_axes: List[str]
     """
     List of names of the diffractometer reciprocal-space (pseudo) axes. Both
     the exact spelling and order are defined by the back-end computation
     library ``engine``.
     MUST match diffractometer to restore.
     """
 
-    constraints: dict[str, DCConstraint]
+    constraints: Dict[str, DCConstraint]
     """
     Limits to be imposed on the real-space axes for operations and
     computations.  Keys must match in the list of ``canonical_axes``.
     """
 
-    samples: dict[str, DCSample]
+    samples: Dict[str, DCSample]
     """
     Crystalline samples (lattice and orientation reflections).
     The sample name is used as the key in the dictionary.
@@ -410,7 +413,7 @@ class DCConfiguration:
     Name of the Python class that defines this diffractometer. (optional)
     """
 
-    other: dict[str, typing.Any] = field(default_factory=dict)
+    other: Dict[str, typing.Any] = field(default_factory=dict)
     """
     *Any* other content goes into this dictionary (comments, unanticipated
     keys, ...) (optional)

hkl/diffract.py

@@ -168,7 +168,7 @@ class (using `calc_kw`) to instantiate a new one.
     lattice_reciprocal = Cpt(
         # fmt: off
         AttributeSignal,
-        attr="calc.sample.reciprocal",
+        attr="calc._cfg_reciprocal",
         doc="Reciprocal lattice",
         # fmt: on
     )
@@ -177,8 +177,8 @@ class (using `calc_kw`) to instantiate a new one.
     UB = Cpt(AttributeSignal, attr="calc.sample.UB", doc="UB matrix")
     # fmt: off
     reflections = Cpt(
-        ArrayAttributeSignal,
-        attr="calc.sample.reflections",
+        AttributeSignal,
+        attr="_reflections",
         doc="Reflections",
     )
     reflections_details = Cpt(
@@ -449,6 +449,11 @@ def engine(self):
     # -- it becomes a problem when someone uses these functions
     # outside of move()
 
+    @property
+    def _reflections(self):
+        """Return the list of reflections as a [[float]]"""
+        return [list(r) for r in self.calc.sample.reflections]
+
     @pseudo_position_argument
     def forward(self, pseudo):
         """

hkl/tests/test_save_restore_UB.py

@@ -1,3 +1,5 @@
+import time
+
 import bluesky.plans as bp
 import databroker
 import numpy.testing
@@ -84,18 +86,16 @@ def test_fourc_orientation_save(cat, RE, fourc):
     # this run will not save orientation information
     _uids = RE(bp.count([det]))
     assert len(_uids) == 1
+    time.sleep(1)
     assert len(cat) == 1
     assert "fourc" not in cat[1].primary.config
 
     # this run _will_ save orientation information
     _uids = RE(bp.count([det, fourc]))
     assert len(_uids) == 1
+    time.sleep(1)
     assert len(cat) == 2
-    xarray_data = cat[2].primary.config["fourc"].read()
-    assert not isinstance(xarray_data, dict)
-    descriptors = xarray_data.to_dict()
-    assert isinstance(descriptors, dict)
-    assert list(descriptors.keys()) == "coords attrs dims data_vars".split()
+
     key_list = """
         _pseudos
         _reals
@@ -108,20 +108,54 @@ def test_fourc_orientation_save(cat, RE, fourc):
         sample_name
         UB
     """.split()
-    for key in key_list:
-        key_name = f"fourc_{key}"
-        assert hasattr(xarray_data, key_name)
-        assert key_name in descriptors["data_vars"]
-
-    assert xarray_data.fourc_class_name == "Fourc"
-    assert xarray_data.fourc_geometry_name == "E4CV"
-    assert xarray_data.fourc_diffractometer_name == "fourc"
-    assert xarray_data.fourc_sample_name == "Si"
-
-    assert len(xarray_data.fourc__pseudos) == 1
-    assert xarray_data.fourc__pseudos[0].values.tolist() == "h k l".split()
-    assert len(xarray_data.fourc__reals) == 1
-    assert xarray_data.fourc__reals.values[0].tolist() == "omega chi phi tth".split()
+
+    if databroker.__version__ < "2.0":
+        conf = cat[_uids[0]].primary.config["fourc"].read()
+        assert not isinstance(conf, dict)
+        descriptors = conf.to_dict()
+        assert isinstance(descriptors, dict)
+        assert list(descriptors.keys()) == "coords attrs dims data_vars".split()
+        for key in key_list:
+            key_name = f"fourc_{key}"
+            assert hasattr(conf, key_name)
+            assert key_name in descriptors["data_vars"]
+
+        assert conf.fourc_class_name == "Fourc"
+        assert conf.fourc_geometry_name == "E4CV"
+        assert conf.fourc_diffractometer_name == "fourc"
+        assert conf.fourc_sample_name == "Si"
+
+        assert len(conf.fourc__pseudos) == 1
+        assert conf.fourc__pseudos[0].values.tolist() == "h k l".split()
+        assert len(conf.fourc__reals) == 1
+        assert conf.fourc__reals.values[0].tolist() == "omega chi phi tth".split()
+
+    else:
+        descriptors = cat[_uids[0]].primary.descriptors
+        assert isinstance(descriptors, list)
+        assert len(descriptors) > 0
+        for descriptor in descriptors:
+            for device, configuration in descriptor.get("configuration", {}).items():
+                assert isinstance(device, str)
+                assert isinstance(configuration, dict)
+                conf = configuration.get("data", {})
+                assert isinstance(conf, dict)
+
+                if device != "fourc":
+                    continue
+                for key in key_list:
+                    key_name = f"fourc_{key}"
+                    assert key_name in conf
+
+                assert conf["fourc_class_name"] == "Fourc"
+                assert conf["fourc_geometry_name"] == "E4CV"
+                assert conf["fourc_diffractometer_name"] == "fourc"
+                assert conf["fourc_sample_name"] == "Si"
+
+                assert len(conf["fourc__pseudos"]) == 3
+                assert conf["fourc__pseudos"] == "h k l".split()
+                assert len(conf["fourc__reals"]) == 4
+                assert conf["fourc__reals"] == "omega chi phi tth".split()
 
 
 def test_fourc_run_orientation_info(cat, RE, fourc):