From 2e20e6ff555d2102277917ff11698f9f576c49ce Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Fri, 14 Jan 2022 12:05:09 +0100 Subject: [PATCH 01/19] Minor fixes, adding ToDo. --- README.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 6c4dbda..a68004f 100644 --- a/README.rst +++ b/README.rst @@ -147,7 +147,7 @@ And in your ``script.jl``, you can intercept the value with .. code-block:: Julia - FIXME FOR YOUR LANGUAGE + ToDo: FIXME FOR YOUR LANGUAGE args <- commandArgs(trailingOnly=TRUE) arg <- args[1] # holds ``"value"`` @@ -173,10 +173,11 @@ The following task executes two Julia scripts which produce different outputs. def task_execute_julia_script(): pass -And the R script includes something like +And the Julia script includes something like -.. code-block:: r +.. code-block:: julia + ToDo: FIXME FOR YOUR LANGUAGE args <- commandArgs(trailingOnly=TRUE) produces <- args[1] # holds the path From cef6170877b9219c96c33d19932df9a272ec4cfb Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Fri, 14 Jan 2022 14:26:22 +0100 Subject: [PATCH 02/19] Fixing test_parallel.py --- tests/test_parallel.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index b904b8b..b08ff7b 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -42,15 +42,13 @@ def task_execute_julia(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) - julia_script = """FIXME FOR YOUR LANGUAGE - Sys.sleep(2) - saveRDS(1, file=paste0(1, ".csv")) + julia_script = """ + write("1.csv", "1") """ tmp_path.joinpath("script_1.jl").write_text(textwrap.dedent(julia_script)) - r_script = """ - Sys.sleep(2) - saveRDS(2, file=paste0(2, ".csv")) + julia_script = """ + write("2.csv", "2") """ tmp_path.joinpath("script_2.jl").write_text(textwrap.dedent(julia_script)) @@ -95,12 +93,10 @@ def task_execute_julia_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) - julia_script = """FIXME FOR YOUR LANGUAGE - Sys.sleep(2) - args <- commandArgs(trailingOnly=TRUE) - number <- args[1] - produces <- args[2] - saveRDS(number, file=produces) + julia_script = """ + number = ARGS[1] + produces = ARGS[2] + write(produces, number) """ tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) From 3edca63e47a8597709b826544562411b6ba63a79 Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Fri, 14 Jan 2022 14:35:58 +0100 Subject: [PATCH 03/19] Fixing test_parametrize.py --- tests/test_parametrize.py | 46 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index 3032882..d5e5a3e 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -8,30 +8,28 @@ @needs_julia @pytest.mark.end_to_end -def test_parametrized_execution_of_r_script(tmp_path): +def test_parametrized_execution_of_jl_script(tmp_path): task_source = """ import pytask @pytask.mark.julia @pytask.mark.parametrize("depends_on, produces", [ - ("script_1.r", "0.txt"), - ("script_2.r", "1.txt"), + ("script_1.jl", "0.txt"), + ("script_2.jl", "1.txt"), ]) - def task_run_r_script(): + def task_run_jl_script(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) for name, content, out in [ - ("script_1.r", "Cities breaking down on a camel's back", "0.txt"), - ("script_2.r", "They just have to go 'cause they don't know whack", "1.txt"), + ("script_1.jl", "Cities breaking down on a camel's back", "0.txt"), + ("script_2.jl", "They just have to go 'cause they don't know whack", "1.txt"), ]: - r_script = f""" - file_descr <- file("{out}") - writeLines(c("{content}"), file_descr) - close(file_descr) + julia_script = f""" + write("{out}", "{content}") """ - tmp_path.joinpath(name).write_text(textwrap.dedent(r_script)) + tmp_path.joinpath(name).write_text(textwrap.dedent(julia_script)) os.chdir(tmp_path) session = main({"paths": tmp_path}) @@ -43,33 +41,33 @@ def task_run_r_script(): @needs_julia @pytest.mark.end_to_end -def test_parametrize_r_options_and_product_paths(tmp_path): +def test_parametrize_jl_options_and_product_paths(tmp_path): task_source = """ import pytask from pathlib import Path SRC = Path(__file__).parent - @pytask.mark.depends_on("script.r") - @pytask.mark.parametrize("produces, r", [ - (SRC / "0.rds", (0, SRC / "0.rds")), (SRC / "1.rds", (1, SRC / "1.rds")) + @pytask.mark.depends_on("script.jl") + @pytask.mark.parametrize("produces, julia", [ + (SRC / "0.csv", (0, SRC / "0.csv")), (SRC / "1.csv", (1, SRC / "1.csv")) ]) - def task_execute_r_script(): + def task_run_jl_script(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - r_script = """ - args <- commandArgs(trailingOnly=TRUE) - number <- args[1] - produces <- args[2] - saveRDS(number, file=produces) + julia_script = """ + number = ARGS[1] + produces = ARGS[2] + write(produces, number) """ - tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) + + tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) os.chdir(tmp_path) session = main({"paths": tmp_path}) assert session.exit_code == 0 - assert tmp_path.joinpath("0.rds").exists() - assert tmp_path.joinpath("1.rds").exists() + assert tmp_path.joinpath("0.csv").exists() + assert tmp_path.joinpath("1.csv").exists() From b3cc4ba702ea1946ff2e1072f03f525e3fa6a0a8 Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Fri, 14 Jan 2022 14:48:00 +0100 Subject: [PATCH 04/19] Adjusting the ReadMe. --- README.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index a68004f..57891aa 100644 --- a/README.rst +++ b/README.rst @@ -147,9 +147,7 @@ And in your ``script.jl``, you can intercept the value with .. code-block:: Julia - ToDo: FIXME FOR YOUR LANGUAGE - args <- commandArgs(trailingOnly=TRUE) - arg <- args[1] # holds ``"value"`` + arg = ARGS[1] # holds ``"value"`` Parametrization @@ -177,9 +175,7 @@ And the Julia script includes something like .. code-block:: julia - ToDo: FIXME FOR YOUR LANGUAGE - args <- commandArgs(trailingOnly=TRUE) - produces <- args[1] # holds the path + produces = ARGS[1] # holds the path If you want to pass different command line arguments to the same Julia script, you have to include the ``@pytask.mark.julia`` decorator in the parametrization just like with From f2d26545b2a3c1157655615a38032b38ccfe626e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 13:55:56 +0000 Subject: [PATCH 05/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- LICENSE | 3 --- setup.cfg | 3 +-- src/pytask_julia/collect.py | 4 +--- tests/test_collect.py | 2 +- tests/test_execute.py | 4 +++- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index a745662..3d13d4b 100644 --- a/LICENSE +++ b/LICENSE @@ -19,6 +19,3 @@ PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - diff --git a/setup.cfg b/setup.cfg index 453692b..f367652 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = pytask-julia +name = pytask_julia description = A Pytask plugin for Julia long_description = file: README.rst long_description_content_type = text/x-rst @@ -18,7 +18,6 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 project_urls = Changelog = https://github.com/pytask-dev/pytask-julia/blob/main/CHANGES.rst Documentation = https://github.com/pytask-dev/pytask-julia diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index 747b4c3..609827a 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -57,9 +57,7 @@ def pytask_collect_task_teardown(session, task): """Perform some checks.""" if get_specific_markers_from_task(task, "julia"): source = _get_node_from_dictionary(task.depends_on, "source") - if isinstance(source, FilePathNode) and source.value.suffix not in [ - ".jl" - ]: + if isinstance(source, FilePathNode) and source.value.suffix not in [".jl"]: raise ValueError( "The first dependency of a Julia task must be the script to be executed." ) diff --git a/tests/test_collect.py b/tests/test_collect.py index 38004f0..b79749e 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -7,9 +7,9 @@ from pytask_julia.collect import _get_node_from_dictionary from pytask_julia.collect import _merge_all_markers from pytask_julia.collect import _prepare_cmd_options +from pytask_julia.collect import julia from pytask_julia.collect import pytask_collect_task from pytask_julia.collect import pytask_collect_task_teardown -from pytask_julia.collect import julia class DummyClass: diff --git a/tests/test_execute.py b/tests/test_execute.py index 81e60da..a789c3d 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -67,7 +67,9 @@ def task_run_jl_script(): and "source" not in depends_on and 0 not in depends_on ): - tmp_path.joinpath("pytask.ini").write_text("[pytask]\njulia_source_key = script") + tmp_path.joinpath("pytask.ini").write_text( + "[pytask]\njulia_source_key = script" + ) os.chdir(tmp_path) session = main({"paths": tmp_path}) From 2e8d4ad3599200c3caca7418f95f588560141111 Mon Sep 17 00:00:00 2001 From: Hans-Martin von Gaudecker Date: Fri, 14 Jan 2022 15:25:27 +0100 Subject: [PATCH 06/19] Make parallel timing meaningful. --- tests/test_parallel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index b08ff7b..0ee7367 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -43,11 +43,13 @@ def task_execute_julia(): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) julia_script = """ + sleep(0.5) write("1.csv", "1") """ tmp_path.joinpath("script_1.jl").write_text(textwrap.dedent(julia_script)) julia_script = """ + sleep(0.5) write("2.csv", "2") """ tmp_path.joinpath("script_2.jl").write_text(textwrap.dedent(julia_script)) @@ -96,6 +98,7 @@ def task_execute_julia_script(): julia_script = """ number = ARGS[1] produces = ARGS[2] + sleep(0.5) write(produces, number) """ tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) From 7f09b9b22af2a055ec6b5f5a3d32a937298aab0b Mon Sep 17 00:00:00 2001 From: Hans-Martin von Gaudecker Date: Fri, 14 Jan 2022 15:56:47 +0100 Subject: [PATCH 07/19] Half a second was too optimistic for GA machines... --- tests/test_parallel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 0ee7367..1a6f0aa 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -43,13 +43,13 @@ def task_execute_julia(): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) julia_script = """ - sleep(0.5) + sleep(2) write("1.csv", "1") """ tmp_path.joinpath("script_1.jl").write_text(textwrap.dedent(julia_script)) julia_script = """ - sleep(0.5) + sleep(2) write("2.csv", "2") """ tmp_path.joinpath("script_2.jl").write_text(textwrap.dedent(julia_script)) @@ -98,7 +98,7 @@ def task_execute_julia_script(): julia_script = """ number = ARGS[1] produces = ARGS[2] - sleep(0.5) + sleep(2) write(produces, number) """ tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) From 5ad6a9d5eddc1297a745b8982360613a64a54263 Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Fri, 14 Jan 2022 20:18:39 +0100 Subject: [PATCH 08/19] [WIP] Fixing the pre-commit issues. --- README.rst | 24 ++++++++++++------------ src/pytask_julia/collect.py | 3 ++- src/pytask_julia/execute.py | 3 ++- tests/test_execute.py | 1 - 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index 57891aa..462ee91 100644 --- a/README.rst +++ b/README.rst @@ -77,8 +77,8 @@ Here is an example where you want to run ``script.julia``. def task_run_jl_script(): pass -Note that, you need to apply the ``@pytask.mark.julia`` marker so that pytask-julia handles the -task. +Note that, you need to apply the ``@pytask.mark.julia`` marker so that pytask-julia +handles the task. If you are wondering why the function body is empty, know that pytask-julia replaces the body with a predefined internal function. See the section on implementation details for @@ -88,8 +88,8 @@ more information. Multiple dependencies and products ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -What happens if a task has more dependencies? Using a list, the Julia script which should be -executed must be found in the first position of the list. +What happens if a task has more dependencies? Using a list, the Julia script which +should be executed must be found in the first position of the list. .. code-block:: python @@ -153,8 +153,8 @@ And in your ``script.jl``, you can intercept the value with Parametrization ~~~~~~~~~~~~~~~ -You can also parametrize the execution of scripts, meaning executing multiple Julia scripts -as well as passing different command line arguments to the same Julia script. +You can also parametrize the execution of scripts, meaning executing multiple Julia +scripts as well as passing different command line arguments to the same Julia script. The following task executes two Julia scripts which produce different outputs. @@ -177,9 +177,9 @@ And the Julia script includes something like produces = ARGS[1] # holds the path -If you want to pass different command line arguments to the same Julia script, you have to -include the ``@pytask.mark.julia`` decorator in the parametrization just like with -``@pytask.mark.depends_on`` and ``@pytask.mark.produces``. +If you want to pass different command line arguments to the same Julia script, you +have to include the ``@pytask.mark.julia`` decorator in the parametrization just like +with ``@pytask.mark.depends_on`` and ``@pytask.mark.produces``. .. code-block:: python @@ -217,9 +217,9 @@ The plugin is a convenient wrapper around to which you can always resort to when the plugin does not deliver functionality you need. -It is not possible to enter a post-mortem debugger when an error happens in the Julia script -or enter the debugger when starting the script. If there exists a solution for that, -hints as well as contributions are highly appreciated. +It is not possible to enter a post-mortem debugger when an error happens in the Julia +script or enter the debugger when starting the script. If there exists a solution for +that, hints as well as contributions are highly appreciated. Changes diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index 609827a..beea8d5 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -59,7 +59,8 @@ def pytask_collect_task_teardown(session, task): source = _get_node_from_dictionary(task.depends_on, "source") if isinstance(source, FilePathNode) and source.value.suffix not in [".jl"]: raise ValueError( - "The first dependency of a Julia task must be the script to be executed." + "The first dependency of a Julia task must be \ + the script to be executed." ) julia_function = _copy_func(run_jl_script) diff --git a/src/pytask_julia/execute.py b/src/pytask_julia/execute.py index 9449a25..664d61c 100644 --- a/src/pytask_julia/execute.py +++ b/src/pytask_julia/execute.py @@ -11,5 +11,6 @@ def pytask_execute_task_setup(task): if get_specific_markers_from_task(task, "julia"): if shutil.which("julia") is None: raise RuntimeError( - "julia is needed to run Julia scripts, but it is not found on your PATH." + "julia is needed to run Julia scripts, \ + but it is not found on your PATH." ) diff --git a/tests/test_execute.py b/tests/test_execute.py index a789c3d..426899d 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -109,7 +109,6 @@ def task_run_jl_script(): @needs_julia @pytest.mark.end_to_end def test_run_jl_script_w_wrong_cmd_option(tmp_path): - """Apparently, RScript simply discards wrong cmd options -- hopefully julia does better.""" task_source = """ import pytask From c545bec5574a34229c3a38757e7e37d7c2cee87c Mon Sep 17 00:00:00 2001 From: Hans-Martin von Gaudecker Date: Sat, 15 Jan 2022 13:12:59 +0100 Subject: [PATCH 09/19] Cheap way around pre-commit failure. --- tests/test_execute.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 426899d..eeba23d 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -24,7 +24,9 @@ class DummyTask: def test_pytask_execute_task_setup(monkeypatch, found_julia, expectation): """Make sure that the task setup raises errors.""" # Act like julia is installed since we do not test this. - monkeypatch.setattr("pytask_julia.execute.shutil.which", lambda x: found_julia) + monkeypatch.setattr( + "pytask_julia.execute.shutil.which", lambda x: found_julia # noqa: U100 + ) task = DummyTask() task.markers = [Mark("julia", (), {})] @@ -98,7 +100,9 @@ def task_run_jl_script(): tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) # Hide julia if available. - monkeypatch.setattr("pytask_julia.execute.shutil.which", lambda x: None) + monkeypatch.setattr( + "pytask_julia.execute.shutil.which", lambda x: None # noqa: U100 + ) session = main({"paths": tmp_path}) From c638c0f14fa508ff27a3af80b3355b7d88ce1625 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 15 Jan 2022 19:34:58 +0100 Subject: [PATCH 10/19] Last fixes. --- .github/workflows/main.yml | 3 +-- .pre-commit-config.yaml | 3 +-- CHANGES.rst | 5 +++-- LICENSE | 1 - README.rst | 15 ++++++++------- environment.yml | 5 ++++- src/pytask_julia/collect.py | 4 ++-- src/pytask_julia/execute.py | 4 ++-- tests/test_execute.py | 8 ++------ tox.ini | 2 +- 10 files changed, 24 insertions(+), 26 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c27b58a..66d68a2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: run-tests: - name: Run tests for ${{ matrix.os }} on ${{ matrix.python-version }} and ${{ matrix.julia-version }} + name: Run tests for ${{ matrix.os }} on ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: @@ -19,7 +19,6 @@ jobs: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.7', '3.8', '3.9'] - julia-version: ['main_version'] steps: - uses: actions/checkout@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 865c7e0..d32ef30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,6 @@ repos: - id: check-merge-conflict - id: check-vcs-permalinks - id: check-yaml - exclude: meta.yaml - id: debug-statements - id: end-of-file-fixer - id: fix-byte-order-marker @@ -33,7 +32,7 @@ repos: rev: v2.21.2 hooks: - id: pyupgrade - args: [--py36-plus] + args: [--py37-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v2.5.0 hooks: diff --git a/CHANGES.rst b/CHANGES.rst index e094b1e..a767c76 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,8 @@ them in reverse chronological order. Releases follow `semantic versioning `_. -0.0.1 - 202X-XX-XX +0.1.0 - 2022-xx-xx ------------------ -- :gh:`01` fixes ... +- :gh:`2` polishes the first release of pytask-julia. (Thanks to :ghuser:`hmgaudecker`, + :ghuser:`hildebrandecon`) diff --git a/LICENSE b/LICENSE index 3d13d4b..3fdd05c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ - MIT License Copyright (c) 2022, Tobias Raabe et al. diff --git a/README.rst b/README.rst index 462ee91..834f883 100644 --- a/README.rst +++ b/README.rst @@ -15,18 +15,18 @@ pytask-julia :alt: PyPI - License :target: https://pypi.org/project/pytask-julia -.. image:: https://img.shields.io/github/workflow/status/hmgaudecker/pytask-julia/main/main - :target: https://github.com/hmgaudecker/pytask-julia/actions?query=branch%3Amain +.. image:: https://img.shields.io/github/workflow/status/pytask-dev/pytask-julia/main/main + :target: https://github.com/pytask-dev/pytask-julia/actions?query=branch%3Amain .. image:: https://readthedocs.org/projects/pytask-julia/badge/?version=latest :target: https://pytask-julia.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -.. image:: https://codecov.io/gh/hmgaudecker/pytask-julia/branch/main/graph/badge.svg - :target: https://codecov.io/gh/hmgaudecker/pytask-julia +.. image:: https://codecov.io/gh/pytask-dev/pytask-julia/branch/main/graph/badge.svg + :target: https://codecov.io/gh/pytask-dev/pytask-julia -.. image:: https://results.pre-commit.ci/badge/github/hmgaudecker/pytask-julia/main.svg - :target: https://results.pre-commit.ci/latest/github/hmgaudecker/pytask-julia/main +.. image:: https://results.pre-commit.ci/badge/github/pytask-dev/pytask-julia/main.svg + :target: https://results.pre-commit.ci/latest/github/pytask-dev/pytask-julia/main :alt: pre-commit.ci status .. image:: https://img.shields.io/badge/code%20style-black-000000.svg @@ -36,7 +36,8 @@ pytask-julia Installation ------------ -pytask-julia is available on `PyPI `_ and `Anaconda.org `_. Install it with +pytask-julia is available on `PyPI `_ and +`Anaconda.org `_. Install it with .. code-block:: console diff --git a/environment.yml b/environment.yml index 88d11bd..c9f6da4 100644 --- a/environment.yml +++ b/environment.yml @@ -5,13 +5,13 @@ channels: - nodefaults dependencies: - - julia - python - pip - setuptools_scm - toml # Package dependencies + # - julia - pytask >=0.1 - pytask-parallel >=0.1 @@ -24,3 +24,6 @@ dependencies: - pytest-cov - pytest-xdist - tox-conda + + - pip: + - -e . diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index beea8d5..f544510 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -59,8 +59,8 @@ def pytask_collect_task_teardown(session, task): source = _get_node_from_dictionary(task.depends_on, "source") if isinstance(source, FilePathNode) and source.value.suffix not in [".jl"]: raise ValueError( - "The first dependency of a Julia task must be \ - the script to be executed." + "The first dependency of a Julia task must be the script to be " + "executed." ) julia_function = _copy_func(run_jl_script) diff --git a/src/pytask_julia/execute.py b/src/pytask_julia/execute.py index 664d61c..8a3bb38 100644 --- a/src/pytask_julia/execute.py +++ b/src/pytask_julia/execute.py @@ -11,6 +11,6 @@ def pytask_execute_task_setup(task): if get_specific_markers_from_task(task, "julia"): if shutil.which("julia") is None: raise RuntimeError( - "julia is needed to run Julia scripts, \ - but it is not found on your PATH." + "julia is needed to run Julia scripts, but it is not found on your " + "PATH." ) diff --git a/tests/test_execute.py b/tests/test_execute.py index eeba23d..307755d 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -59,9 +59,7 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = """ - write("out.txt", "So, so you think you can tell heaven from hell?") - """ + julia_script = "write('out.txt', 'So, so you think you can tell heaven from hell?')" tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) if ( @@ -94,9 +92,7 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = """ - write("out.txt", "So, so you think you can tell heaven from hell?") - """ + julia_script = "write('out.txt', 'So, so you think you can tell heaven from hell?')" tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) # Hide julia if available. diff --git a/tox.ini b/tox.ini index 04e5a67..970800d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pytest, pre-commit +envlist = pytest skipsdist = True skip_missing_interpreters = True passenv = PY_IGNORE_IMPORTMISMATCH From 2a99009acb165fab66503a277ba4e9ab73f5f21a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 15 Jan 2022 23:55:10 +0100 Subject: [PATCH 11/19] Fix tests. --- environment.yml | 2 +- tests/test_execute.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index c9f6da4..a2b7531 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - toml # Package dependencies - # - julia + - julia - pytask >=0.1 - pytask-parallel >=0.1 diff --git a/tests/test_execute.py b/tests/test_execute.py index 307755d..aa2ade2 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -59,7 +59,7 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = "write('out.txt', 'So, so you think you can tell heaven from hell?')" + julia_script = 'write("out.txt", "So, so you think you can tell heaven from hell?")' tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) if ( @@ -92,7 +92,7 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = "write('out.txt', 'So, so you think you can tell heaven from hell?')" + julia_script = 'write("out.txt", "So, so you think you can tell heaven from hell?")' tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) # Hide julia if available. From 51edea355c5495f81f8aad42389267303a34e765 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jan 2022 00:04:50 +0100 Subject: [PATCH 12/19] Increase sleep in paralell tests. --- tests/test_parallel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 1a6f0aa..ad264c3 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -43,13 +43,13 @@ def task_execute_julia(): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) julia_script = """ - sleep(2) + sleep(4) write("1.csv", "1") """ tmp_path.joinpath("script_1.jl").write_text(textwrap.dedent(julia_script)) julia_script = """ - sleep(2) + sleep(4) write("2.csv", "2") """ tmp_path.joinpath("script_2.jl").write_text(textwrap.dedent(julia_script)) @@ -98,7 +98,7 @@ def task_execute_julia_script(): julia_script = """ number = ARGS[1] produces = ARGS[2] - sleep(2) + sleep(4) write(produces, number) """ tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) From 5249296824180e4edf1752d3ace2211c4f7f5a52 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jan 2022 21:33:34 +0100 Subject: [PATCH 13/19] remove julia reference in github actions. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 66d68a2..1d95a9d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: run: tox -e pytest -- -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto - name: Upload coverage report for unit tests and doctests. - if: runner.os == 'Linux' && matrix.python-version == '3.8' && matrix.julia-version == 'main_version' + if: runner.os == 'Linux' && matrix.python-version == '3.8' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F unit -c @@ -47,11 +47,11 @@ jobs: run: tox -e pytest -- -m end_to_end --cov=./ --cov-report=xml -n auto - name: Upload coverage reports of end-to-end tests. - if: runner.os == 'Linux' && matrix.python-version == '3.8' && matrix.julia-version == 'main_version' + if: runner.os == 'Linux' && matrix.python-version == '3.8' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F end_to_end -c - name: Validate codecov.yml - if: runner.os == 'Linux' && matrix.python-version == '3.8' && matrix.julia-version == 'main_version' + if: runner.os == 'Linux' && matrix.python-version == '3.8' shell: bash -l {0} run: cat codecov.yml | curl --data-binary @- https://codecov.io/validate From e2e6677cf716377c65b23f1e1889320d0bef7ef9 Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Mon, 17 Jan 2022 10:17:35 +0100 Subject: [PATCH 14/19] [WIP] Test for cmd line options. --- tests/test_execute.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_execute.py b/tests/test_execute.py index aa2ade2..dc3e532 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -130,3 +130,30 @@ def task_run_jl_script(): session = main({"paths": tmp_path}) assert session.exit_code == 0 + + +@needs_julia +@pytest.mark.end_to_end +def test_check_passing_cmd_line_options(tmp_path): + task_source = """ + import pytask + + @pytask.mark.julia("-t 4") + @pytask.mark.depends_on("script.jl") + @pytask.mark.produces("out.txt") + def task_run_jl_script(): + pass + + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + + julia_script = """ + write("out.txt", "So, so you think you can tell heaven from hell?") + @assert Threads.nthreads() == 1 + """ + tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) + + os.chdir(tmp_path) + session = main({"paths": tmp_path}) + + assert session.exit_code == 0 From b2113aa54ee9ea0d3c88861bea7978870f0bde2a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 18 Jan 2022 20:05:33 +0100 Subject: [PATCH 15/19] Implement suggestion and align readme. --- README.rst | 39 ++++++++++++++++++++++++++++++----- src/pytask_julia/collect.py | 41 +++++++++++++++++++++++++++++++++++-- tests/test_collect.py | 23 +++++++++++++-------- tests/test_execute.py | 4 ++-- tests/test_parallel.py | 3 ++- tests/test_parametrize.py | 3 ++- 6 files changed, 93 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 834f883..8d0d55a 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,13 @@ typing the following on the command line $ julia -h -If an error is shown instead of a help page, you can install Julia .... +If an error is shown instead of a help page, you can install Julia on Unix systems with + +.. code-block:: console + + $ conda install -c conda-forge julia + +or choose one of the installers on this `page `_. Usage @@ -133,12 +139,18 @@ for a ``"source"`` key in the dictionary and, secondly, under the key ``0``. Command Line Arguments ~~~~~~~~~~~~~~~~~~~~~~ -The decorator can be used to pass command line arguments to ``julia``. See the -following example. +The decorator can be used to pass command line arguments to ``julia``. An important +detail is that you need to differentiate between options passed to the Julia executable +and arguments passed to the script. + +First, pass options to the executable, then, use ``"--"`` as a separator, and after that +arguments to the script. + +The following shows how to pass both with the decorator. .. code-block:: python - @pytask.mark.julia("value") + @pytask.mark.julia(("--threads", "2", "--", "value")) @pytask.mark.depends_on("script.jl") @pytask.mark.produces("out.csv") def task_run_jl_script(): @@ -150,6 +162,20 @@ And in your ``script.jl``, you can intercept the value with arg = ARGS[1] # holds ``"value"`` +If you pass only of of them, either options for the executable or arguments to the +script, you still need to include the separator. + +.. code-block:: python + + @python.mark.julia(("--verbose", "--")) # for options for the executable. + def task_func(): + ... + + + @python.mark.julia(("--", "value")) # for arguments for the script. + def task_func(): + ... + Parametrization ~~~~~~~~~~~~~~~ @@ -187,7 +213,10 @@ with ``@pytask.mark.depends_on`` and ``@pytask.mark.produces``. @pytask.mark.depends_on("script.jl") @pytask.mark.parametrize( "produces, julia", - [(BLD / "output_1.csv", "1"), (BLD / "output_2.csv", "2")], + [ + (BLD / "output_1.csv", ("--", "1")), + (BLD / "output_2.csv", ("--", "2")), + ], ) def task_execute_julia_script(): pass diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index f544510..474d62c 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -15,6 +15,10 @@ from _pytask.parametrize import _copy_func +_SEPARATOR = "--" +"""str: Separates options for the Julia executable and arguments to the file.""" + + def julia(options: Optional[Union[str, Iterable[str]]] = None): """Specify command line options for Julia. @@ -24,8 +28,9 @@ def julia(options: Optional[Union[str, Iterable[str]]] = None): One or multiple command line options passed to Julia. """ - options = [] if options is None else _to_list(options) + options = [_SEPARATOR] if options is None else _to_list(options) options = [str(i) for i in options] + return options @@ -90,6 +95,24 @@ def _merge_all_markers(task): return mark +_ERROR_MSG_MISSING_SEPARATOR = f"""The inputs of the @pytask.mark.julia decorator do not +contain the separator to differentiate between options for the julia executable and +arguments to the script. This was passed to the decorator: + +{{}} + +Construct the inputs to the decorator should contain, first, options to the executable, +secondly, the separator - '--' -, thirdly, arguments to the script. + +Here is an example: + +@pytask.mark.julia(("--threads", "1", {_SEPARATOR}, "input.file")) + +Even if you do not need the left or the right side of the decorator, you must put the +separator at the correct position. +""" + + def _prepare_cmd_options(session, task, args): """Prepare the command line arguments to execute the do-file. @@ -100,7 +123,21 @@ def _prepare_cmd_options(session, task, args): source = _get_node_from_dictionary( task.depends_on, session.config["julia_source_key"] ) - return ["julia", source.path.as_posix(), *args] + + if _SEPARATOR not in args: + raise ValueError(_ERROR_MSG_MISSING_SEPARATOR.format(args)) + else: + idx_sep = args.index(_SEPARATOR) + executable_options = args[:idx_sep] + script_arguments = args[idx_sep + 1 :] + + return [ + "julia", + *executable_options, + _SEPARATOR, + source.path.as_posix(), + *script_arguments, + ] def _to_list(scalar_or_iter): diff --git a/tests/test_collect.py b/tests/test_collect.py index b79749e..0257ddf 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -24,7 +24,7 @@ def task_dummy(): @pytest.mark.parametrize( "julia_args, expected", [ - (None, []), + (None, ["--"]), ("--some-option", ["--some-option"]), (["--a", "--b"], ["--a", "--b"]), ], @@ -123,15 +123,20 @@ def test_get_node_from_dictionary(obj, key, expected): @pytest.mark.unit @pytest.mark.parametrize( - "args", + "args, expectation, expected", [ - [], - ["a"], - ["a", "b"], + (("--"), does_not_raise(), ["julia", "--", "script.jl"]), + ( + ("--verbose", "--"), + does_not_raise(), + ["julia", "--verbose", "--", "script.jl"], + ), + (("--", "seed"), does_not_raise(), ["julia", "--", "script.jl", "seed"]), + (("--verbose",), pytest.raises(ValueError, match="The inputs"), None), ], ) @pytest.mark.parametrize("julia_source_key", ["source", "script"]) -def test_prepare_cmd_options(args, julia_source_key): +def test_prepare_cmd_options(args, expectation, expected, julia_source_key): session = DummyClass() session.config = {"julia_source_key": julia_source_key} @@ -141,6 +146,6 @@ def test_prepare_cmd_options(args, julia_source_key): task.depends_on = {julia_source_key: node} task.name = "task" - result = _prepare_cmd_options(session, task, args) - - assert result == ["julia", "script.jl", *args] + with expectation: + result = _prepare_cmd_options(session, task, args) + assert result == expected diff --git a/tests/test_execute.py b/tests/test_execute.py index aa2ade2..c5673d9 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -112,7 +112,7 @@ def test_run_jl_script_w_wrong_cmd_option(tmp_path): task_source = """ import pytask - @pytask.mark.julia("--wrong-flag") + @pytask.mark.julia(("--wrong-flag", "--")) @pytask.mark.depends_on("script.jl") @pytask.mark.produces("out.txt") def task_run_jl_script(): @@ -129,4 +129,4 @@ def task_run_jl_script(): os.chdir(tmp_path) session = main({"paths": tmp_path}) - assert session.exit_code == 0 + assert session.exit_code == 1 diff --git a/tests/test_parallel.py b/tests/test_parallel.py index ad264c3..df8c91d 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -88,7 +88,8 @@ def test_parallel_parametrization_over_source_file(runner, tmp_path): @pytask.mark.depends_on("script.jl") @pytask.mark.parametrize("produces, julia", [ - (SRC / "0.csv", (1, SRC / "0.csv")), (SRC / "1.csv", (1, SRC / "1.csv")) + (SRC / "0.csv", ("--", 1, SRC / "0.csv")), + (SRC / "1.csv", ("--", 1, SRC / "1.csv")), ]) def task_execute_julia_script(): pass diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index d5e5a3e..1f11dc8 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -50,7 +50,8 @@ def test_parametrize_jl_options_and_product_paths(tmp_path): @pytask.mark.depends_on("script.jl") @pytask.mark.parametrize("produces, julia", [ - (SRC / "0.csv", (0, SRC / "0.csv")), (SRC / "1.csv", (1, SRC / "1.csv")) + (SRC / "0.csv", ("--", 0, SRC / "0.csv")), + (SRC / "1.csv", ("--", 1, SRC / "1.csv")), ]) def task_run_jl_script(): pass From 7f31d5cab93de761dd7c33e8080055ab314ad696 Mon Sep 17 00:00:00 2001 From: Sebastian Hildebrand Date: Wed, 19 Jan 2022 08:56:10 +0100 Subject: [PATCH 16/19] Extending cmd line options test. --- tests/test_execute.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_execute.py b/tests/test_execute.py index 3018156..3b645a3 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -157,3 +157,27 @@ def task_run_jl_script(): session = main({"paths": tmp_path}) assert session.exit_code == 0 + + task_source = """ + import pytask + + @pytask.mark.julia(("--threads", "1", "--")) + @pytask.mark.depends_on("script.jl") + @pytask.mark.produces("out.txt") + def task_run_jl_script(): + pass + + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + + julia_script = """ + write("out.txt", "So, so you think you can tell heaven from hell?") + @assert Threads.nthreads() == 4 + """ + tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) + + os.chdir(tmp_path) + session = main({"paths": tmp_path}) + + assert session.exit_code == 1 + From 9ce4f300b1cfd4b8436eeb78ab090c4334e32470 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 07:56:58 +0000 Subject: [PATCH 17/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_execute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 3b645a3..f89fa07 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -180,4 +180,3 @@ def task_run_jl_script(): session = main({"paths": tmp_path}) assert session.exit_code == 1 - From 7d38f89cf0425f5b762f3c76f69f8285aac49c9f Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 19 Jan 2022 19:01:08 +0100 Subject: [PATCH 18/19] Integrate all review comments. --- README.rst | 22 +++++++++-- src/pytask_julia/collect.py | 4 +- tests/test_execute.py | 78 ++++++++++++++----------------------- 3 files changed, 51 insertions(+), 53 deletions(-) diff --git a/README.rst b/README.rst index 8d0d55a..2849e16 100644 --- a/README.rst +++ b/README.rst @@ -144,7 +144,7 @@ detail is that you need to differentiate between options passed to the Julia exe and arguments passed to the script. First, pass options to the executable, then, use ``"--"`` as a separator, and after that -arguments to the script. +arguments to the script. Provide all arguments in a tuple or a list as below. The following shows how to pass both with the decorator. @@ -156,6 +156,12 @@ The following shows how to pass both with the decorator. def task_run_jl_script(): pass +which executes the something similar to the following on the command line. + +.. code-block:: console + + $ julia --threads 2 -- value + And in your ``script.jl``, you can intercept the value with .. code-block:: Julia @@ -167,15 +173,25 @@ script, you still need to include the separator. .. code-block:: python - @python.mark.julia(("--verbose", "--")) # for options for the executable. + @pytask.mark.julia(("--verbose", "--")) # for options for the executable. + @pytask.mark.depends_on("script.jl") def task_func(): ... - @python.mark.julia(("--", "value")) # for arguments for the script. + @pytask.mark.julia(("--", "value")) # for arguments for the script. + @pytask.mark.depends_on("script.jl") def task_func(): ... +The corresponding commands on the command line are + +.. code-block:: console + + $ julia --verbose -- script.jl + + $ julia -- script.jl value + Parametrization ~~~~~~~~~~~~~~~ diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index 474d62c..eb7468e 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -102,11 +102,11 @@ def _merge_all_markers(task): {{}} Construct the inputs to the decorator should contain, first, options to the executable, -secondly, the separator - '--' -, thirdly, arguments to the script. +secondly, the separator - "{_SEPARATOR}" -, thirdly, arguments to the script. Here is an example: -@pytask.mark.julia(("--threads", "1", {_SEPARATOR}, "input.file")) +@pytask.mark.julia(("--threads", "1", "{_SEPARATOR}", "input.file")) Even if you do not need the left or the right side of the decorator, you must put the separator at the correct position. diff --git a/tests/test_execute.py b/tests/test_execute.py index f89fa07..2acd442 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -5,7 +5,7 @@ import pytest from _pytask.mark import Mark from conftest import needs_julia -from pytask import main +from pytask import cli, main from pytask_julia.execute import pytask_execute_task_setup @@ -40,18 +40,23 @@ def test_pytask_execute_task_setup(monkeypatch, found_julia, expectation): @pytest.mark.parametrize( "depends_on", [ - "'script.jl'", + "script.jl", {"source": "script.jl"}, {0: "script.jl"}, {"script": "script.jl"}, ], ) -def test_run_jl_script(tmp_path, depends_on): +def test_run_jl_script(runner, tmp_path, depends_on): + if isinstance(depends_on, str): + full_depends_on = "'" + (tmp_path / depends_on).as_posix() + "'" + else: + full_depends_on = {k: (tmp_path / v).as_posix() for k, v in depends_on.items()} + task_source = f""" import pytask @pytask.mark.julia - @pytask.mark.depends_on({depends_on}) + @pytask.mark.depends_on({full_depends_on}) @pytask.mark.produces("out.txt") def task_run_jl_script(): pass @@ -59,7 +64,8 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = 'write("out.txt", "So, so you think you can tell heaven from hell?")' + out = tmp_path.joinpath("out.txt").as_posix() + julia_script = f'write("{out}", "So, so you think you can tell heaven from hell?")' tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) if ( @@ -71,10 +77,9 @@ def task_run_jl_script(): "[pytask]\njulia_source_key = script" ) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 + assert result.exit_code == 0 assert tmp_path.joinpath("out.txt").exists() @@ -92,7 +97,8 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = 'write("out.txt", "So, so you think you can tell heaven from hell?")' + out = tmp_path.joinpath("out.txt").as_posix() + julia_script = f'write("{out}", "So, so you think you can tell heaven from hell?")' tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) # Hide julia if available. @@ -108,7 +114,7 @@ def task_run_jl_script(): @needs_julia @pytest.mark.end_to_end -def test_run_jl_script_w_wrong_cmd_option(tmp_path): +def test_run_jl_script_w_wrong_cmd_option(runner, tmp_path): task_source = """ import pytask @@ -121,47 +127,23 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = """ - write("out.txt", "So, so you think you can tell heaven from hell?") - """ + out = tmp_path.joinpath("out.txt").as_posix() + julia_script = f'write("{out}", "So, so you think you can tell heaven from hell?")' tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 1 + assert result.exit_code == 1 @needs_julia @pytest.mark.end_to_end -def test_check_passing_cmd_line_options(tmp_path): - task_source = """ - import pytask - - @pytask.mark.julia(("--threads", "4", "--")) - @pytask.mark.depends_on("script.jl") - @pytask.mark.produces("out.txt") - def task_run_jl_script(): - pass - - """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - - julia_script = """ - write("out.txt", "So, so you think you can tell heaven from hell?") - @assert Threads.nthreads() == 4 - """ - tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) - - os.chdir(tmp_path) - session = main({"paths": tmp_path}) - - assert session.exit_code == 0 - - task_source = """ +@pytest.mark.parametrize('n_threads', [2, 3]) +def test_check_passing_cmd_line_options(runner, tmp_path, n_threads): + task_source = f""" import pytask - @pytask.mark.julia(("--threads", "1", "--")) + @pytask.mark.julia(("--threads", "{n_threads}", "--")) @pytask.mark.depends_on("script.jl") @pytask.mark.produces("out.txt") def task_run_jl_script(): @@ -170,13 +152,13 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - julia_script = """ - write("out.txt", "So, so you think you can tell heaven from hell?") - @assert Threads.nthreads() == 4 + out = tmp_path.joinpath("out.txt").as_posix() + julia_script = f""" + write("{out}", "So, so you think you can tell heaven from hell?") + @assert Threads.nthreads() == {n_threads} """ tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 1 + assert result.exit_code == 0 From 4ff5c48b85be6dfcfc78b7ad47d814820a2e35c1 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 19 Jan 2022 19:03:38 +0100 Subject: [PATCH 19/19] Fix. --- tests/test_execute.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 2acd442..7841499 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1,11 +1,11 @@ -import os import textwrap from contextlib import ExitStack as does_not_raise # noqa: N813 import pytest from _pytask.mark import Mark from conftest import needs_julia -from pytask import cli, main +from pytask import cli +from pytask import main from pytask_julia.execute import pytask_execute_task_setup @@ -138,7 +138,7 @@ def task_run_jl_script(): @needs_julia @pytest.mark.end_to_end -@pytest.mark.parametrize('n_threads', [2, 3]) +@pytest.mark.parametrize("n_threads", [2, 3]) def test_check_passing_cmd_line_options(runner, tmp_path, n_threads): task_source = f""" import pytask