diff --git a/conda-env.yml b/conda-env.yml index ee5fc5facb..854baa68f8 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -10,7 +10,7 @@ dependencies: - nibabel>=3.2.0,<4.1 - nilearn>=0.9.0,<=0.10.0 - sqlalchemy>=1.4.27,<= 1.5.0 - - pyyaml>=5.1.2,<7.0 + - ruamel.yaml=0.17.* - h5py=3.8.* - seaborn=0.11.* - Sphinx=5.3.* diff --git a/docs/changes/newsfragments/223.bugfix b/docs/changes/newsfragments/223.bugfix new file mode 100644 index 0000000000..a709cb2e00 --- /dev/null +++ b/docs/changes/newsfragments/223.bugfix @@ -0,0 +1 @@ +Enable YAML 1.2 support and allow multiline strings in YAML which would not work earlier by `Synchon Mandal`_ \ No newline at end of file diff --git a/docs/changes/newsfragments/223.enh b/docs/changes/newsfragments/223.enh new file mode 100644 index 0000000000..54e3befe58 --- /dev/null +++ b/docs/changes/newsfragments/223.enh @@ -0,0 +1 @@ +Use ``ruamel.yaml`` instead of ``pyyaml`` as YAML I/O library by `Synchon Mandal`_ \ No newline at end of file diff --git a/docs/faq.rst b/docs/faq.rst index d933e84a74..0a7a7a5dc7 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -23,7 +23,7 @@ The following steps are specific to VSCode and you can choose to go with it: .. code-block:: bash - conda env create -n -f conda-env.yml python=3.10 + conda env create -n -f conda-env.yml conda activate The ``conda-env.yml`` can be found at the root of the repository. diff --git a/docs/installation.rst b/docs/installation.rst index d9ab3a31ea..e6611abbb4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -16,7 +16,7 @@ junifer is compatible with `Python`_ >= 3.8 and requires the following packages: * ``nibabel>=3.2.0,<4.1`` * ``nilearn>=0.9.0,<=0.10.0`` * ``sqlalchemy>=1.4.27,<= 1.5.0`` -* ``pyyaml>=5.1.2,<7.0`` +* ``ruamel.yaml>=0.17,<0.18`` * ``h5py>=3.8.0,<3.9`` Depending on the installation method, these packages might be installed diff --git a/docs/maintaining.rst b/docs/maintaining.rst index d9a29e68ac..2c883005c4 100644 --- a/docs/maintaining.rst +++ b/docs/maintaining.rst @@ -16,7 +16,7 @@ This plugin reads the latest tagged version from git and automatically increments the *MICRO* segment and appends *devN*. This is considered a pre-release. -The CI scripts will publish every tag with the format *v.X.Y.Z* to Pypi as +The CI scripts will publish every tag with the format *v.X.Y.Z* to PyPI as version "X.Y.Z". Additionally, for every push to main, it will be published as pre-release to PyPI. diff --git a/junifer/api/cli.py b/junifer/api/cli.py index c24482b8c4..d40cc9c684 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -11,7 +11,6 @@ from typing import Dict, List, Union import click -import yaml from ..utils.logging import ( configure_logging, @@ -29,6 +28,7 @@ _get_junifer_version, _get_python_information, _get_system_information, + yaml, ) @@ -275,7 +275,7 @@ def wtf(long_: bool) -> None: "system": _get_system_information(), "environment": _get_environment_information(long_=long_), } - click.echo(yaml.dump(report, sort_keys=False)) + click.echo(yaml.dump(report, stream=sys.stdout)) @cli.command() diff --git a/junifer/api/decorators.py b/junifer/api/decorators.py index eb2f0bda16..87ea51f89e 100644 --- a/junifer/api/decorators.py +++ b/junifer/api/decorators.py @@ -4,6 +4,7 @@ # Leonard Sasse # Synchon Mandal # License: AGPL + from typing import Type from ..pipeline.registry import register diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 6c1a2a7cce..32cc2f8938 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -12,8 +12,6 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Union -import yaml - from ..datagrabber.base import BaseDataGrabber from ..markers.base import BaseMarker from ..markers.collection import MarkerCollection @@ -22,6 +20,7 @@ from ..storage.base import BaseFeatureStorage from ..utils import logger, raise_error from ..utils.fs import make_executable +from .utils import yaml def _get_datagrabber(datagrabber_config: Dict) -> BaseDataGrabber: @@ -270,8 +269,7 @@ def queue( yaml_config = jobdir / "config.yaml" logger.info(f"Writing YAML config to {str(yaml_config.absolute())}") - with open(yaml_config, "w") as f: - f.write(yaml.dump(config)) + yaml.dump(config, stream=yaml_config) # Get list of elements if elements is None: diff --git a/junifer/api/parser.py b/junifer/api/parser.py index 8e96b365e9..fe5301439b 100644 --- a/junifer/api/parser.py +++ b/junifer/api/parser.py @@ -10,9 +10,8 @@ from pathlib import Path from typing import Dict, Union -import yaml - from ..utils.logging import logger, raise_error +from .utils import yaml def parse_yaml(filepath: Union[str, Path]) -> Dict: @@ -38,8 +37,7 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict: if not filepath.exists(): raise_error(f"File does not exist: {str(filepath.absolute())}") # Filepath reading - with open(filepath, "r") as f: - contents = yaml.safe_load(f) + contents = yaml.load(filepath) if "elements" in contents: if contents["elements"] is None: raise_error( diff --git a/junifer/api/tests/test_api_utils.py b/junifer/api/tests/test_api_utils.py index 33dd4db425..57b67624a6 100644 --- a/junifer/api/tests/test_api_utils.py +++ b/junifer/api/tests/test_api_utils.py @@ -40,7 +40,7 @@ def test_get_dependency_information_short() -> None: "nibabel", "nilearn", "sqlalchemy", - "yaml", + "ruamel.yaml", ] @@ -58,7 +58,7 @@ def test_get_dependency_information_long() -> None: "nibabel", "nilearn", "sqlalchemy", - "yaml", + "ruamel.yaml", ]: assert key in dependency_information_keys diff --git a/junifer/api/tests/test_cli.py b/junifer/api/tests/test_cli.py index 8e90dc765e..aa01285e4e 100644 --- a/junifer/api/tests/test_cli.py +++ b/junifer/api/tests/test_cli.py @@ -8,12 +8,18 @@ from typing import Tuple import pytest -import yaml from click.testing import CliRunner +from ruamel.yaml import YAML from junifer.api.cli import collect, run, selftest, wtf +# Configure YAML class +yaml = YAML() +yaml.default_flow_style = False +yaml.allow_unicode = True +yaml.indent(mapping=2, sequence=4, offset=2) + # Create click test runner runner = CliRunner() @@ -33,19 +39,17 @@ def test_run_and_collect_commands( # Get test config infile = Path(__file__).parent / "data" / "gmd_mean.yaml" # Read test config - with open(infile, mode="r") as f: - contents = yaml.safe_load(f) + contents = yaml.load(infile) # Working directory workdir = tmp_path / "workdir" - contents["workdir"] = str(workdir.absolute()) + contents["workdir"] = str(workdir.resolve()) # Output directory outdir = tmp_path / "outdir" # Storage - contents["storage"]["uri"] = str(outdir.absolute()) + contents["storage"]["uri"] = str(outdir.resolve()) # Write new test config outfile = tmp_path / "in.yaml" - with open(outfile, mode="w") as f: - yaml.dump(contents, f) + yaml.dump(contents, stream=outfile) # Run command arguments run_args = [ str(outfile.absolute()), diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 79c4dd9e58..b345a0ac7d 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -11,7 +11,7 @@ from typing import List, Tuple, Union import pytest -import yaml +from ruamel.yaml import YAML import junifer.testing.registry # noqa: F401 from junifer.api.functions import collect, queue, run @@ -19,6 +19,12 @@ from junifer.pipeline.registry import build +# Configure YAML class +yaml = YAML() +yaml.default_flow_style = False +yaml.allow_unicode = True +yaml.indent(mapping=2, sequence=4, offset=2) + # Define datagrabber datagrabber = { "kind": "OasisVBMTestingDataGrabber", @@ -231,6 +237,7 @@ def test_run_and_collect(tmp_path: Path) -> None: def test_queue_correct_yaml_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: """Test proper YAML config generation for queueing. @@ -240,32 +247,37 @@ def test_queue_correct_yaml_config( The path to the test directory. monkeypatch : pytest.MonkeyPatch The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. """ with monkeypatch.context() as m: m.chdir(tmp_path) - queue( - config={ - "with": "junifer.testing.registry", - "workdir": str(Path(tmp_path).resolve()), - "datagrabber": datagrabber, - "markers": markers, - "storage": storage, - "env": { - "kind": "conda", - "name": "junifer", + with caplog.at_level(logging.INFO): + queue( + config={ + "with": "junifer.testing.registry", + "workdir": str(tmp_path.resolve()), + "datagrabber": datagrabber, + "markers": markers, + "storage": {"kind": "SQLiteFeatureStorage"}, + "env": { + "kind": "conda", + "name": "junifer", + }, + "mem": "8G", }, - "mem": "8G", - }, - kind="HTCondor", - jobname="yaml_config_gen_check", - ) + kind="HTCondor", + jobname="yaml_config_gen_check", + ) + assert "Creating job in" in caplog.text + assert "Writing YAML config to" in caplog.text + assert "Queue done" in caplog.text generated_config_yaml_path = Path( tmp_path / "junifer_jobs" / "yaml_config_gen_check" / "config.yaml" ) - with open(generated_config_yaml_path, "r") as f: - yaml_config = yaml.unsafe_load(f) + yaml_config = yaml.load(generated_config_yaml_path) # Check for correct YAML config generation assert all( key in yaml_config.keys() diff --git a/junifer/api/utils.py b/junifer/api/utils.py index 0e2ea1001d..b4d8ba13b5 100644 --- a/junifer/api/utils.py +++ b/junifer/api/utils.py @@ -9,10 +9,19 @@ from importlib.metadata import distribution from typing import Dict +from ruamel.yaml import YAML + from .._version import __version__ from ..utils.logging import get_versions +# Configure YAML class once for further use +yaml = YAML() +yaml.default_flow_style = False +yaml.allow_unicode = True +yaml.indent(mapping=2, sequence=4, offset=2) + + def _get_junifer_version() -> Dict[str, str]: """Get junifer version information. @@ -71,16 +80,13 @@ def _get_dependency_information(long_: bool) -> Dict[str, str]: # Get dependencies for junifer dist = distribution("junifer") # Compile regex pattern - re_pattern = re.compile("[a-z-]+") + re_pattern = re.compile("[a-z-_.]+") for pkg_with_version in dist.requires: # type: ignore # Perform regex search matches = re.findall(pattern=re_pattern, string=pkg_with_version) - # Fix issue with PyYAML name registration - if matches[0] == "pyyaml": - key = "yaml" - else: - key = matches[0] + # Extract package name + key = matches[0] if key in dependency_versions.keys(): # Check if pkg part of optional dependencies diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index dd7bab3bda..75e632533a 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -29,7 +29,7 @@ ("nibabel", "4.1"), ("nilearn", "0.10.0"), ("sqlalchemy", "1.5.0"), - ("pyyaml", "7.0"), + ("ruamel.yaml", "0.18.0"), ], ) def test_get_dependency_version(dependency: str, max_version: str) -> None: diff --git a/junifer/utils/logging.py b/junifer/utils/logging.py index 5e06dcd480..922b248a8f 100644 --- a/junifer/utils/logging.py +++ b/junifer/utils/logging.py @@ -90,7 +90,9 @@ def get_versions() -> Dict: """ module_versions = {} for name, module in sys.modules.items(): - if "." in name: + # Bypassing sub-modules of packages and + # allowing ruamel.yaml + if "." in name and name != "ruamel.yaml": continue if name in ["_curses"]: continue diff --git a/pyproject.toml b/pyproject.toml index db1ff6a465..f05b5b7c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "nibabel>=3.2.0,<4.1", "nilearn>=0.9.0,<=0.10.0", "sqlalchemy>=1.4.27,<= 1.5.0", - "pyyaml>=5.1.2,<7.0", + "ruamel.yaml>=0.17,<0.18", "importlib_metadata; python_version < '3.10'", "h5py>=3.8.0,<3.9", ]