Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ repos:
args: [--branch, main]
- id: trailing-whitespace
- repo: https://github.com/pre-commit/pygrep-hooks
rev: v1.9.0
rev: v1.10.0
hooks:
- id: python-check-blanket-noqa
- id: python-check-mock-methods
Expand All @@ -43,11 +43,11 @@ repos:
hooks:
- id: black
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.215
rev: v0.0.223
hooks:
- id: ruff
- repo: https://github.com/dosisod/refurb
rev: v1.9.1
rev: v1.10.0
hooks:
- id: refurb
args: [--ignore, FURB126]
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,12 @@ extend-ignore = [
"EM", # flake8-errmsg
"ANN401", # flake8-annotate typing.Any
"PD", # pandas-vet
"COM812", # trailing comma missing, but black takes care of that
]


[tool.ruff.per-file-ignores]
"tests/*" = ["D", "ANN"]
"tests/*" = ["D", "ANN", "PLR2004"]


[tool.ruff.pydocstyle]
Expand Down
23 changes: 16 additions & 7 deletions src/pytask_julia/collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from pytask import Task
from pytask_julia.serialization import SERIALIZERS
from pytask_julia.shared import julia
from pytask_julia.shared import JULIA_SCRIPT_KEY
from pytask_julia.shared import parse_relative_path


Expand All @@ -27,7 +28,10 @@


def run_jl_script(
script: Path, options: list[str], serialized: Path, project: list[str]
script: Path,
options: list[str],
serialized: Path,
project: list[str],
) -> None:
"""Run a Julia script."""
cmd = ["julia"] + options + project + [_SEPARATOR, str(script), str(serialized)]
Expand All @@ -37,7 +41,10 @@ def run_jl_script(

@hookimpl
def pytask_collect_task(
session: Session, path: Path, name: str, obj: Any
session: Session,
path: Path,
name: str,
obj: Any,
) -> Task | None:
"""Collect a task which is a function.

Expand All @@ -59,7 +66,7 @@ def pytask_collect_task(
if len(marks) > 1:
raise ValueError(
f"Task {name!r} has multiple @pytask.mark.julia marks, but only one is "
"allowed."
"allowed.",
)

julia_mark = _parse_julia_mark(
Expand Down Expand Up @@ -90,21 +97,23 @@ def pytask_collect_task(
)

script_node = session.hook.pytask_collect_node(
session=session, path=path, node=script
session=session,
path=path,
node=script,
)

if isinstance(task.depends_on, dict):
task.depends_on["__script"] = script_node
task.depends_on[JULIA_SCRIPT_KEY] = script_node
task.attributes["julia_keep_dict"] = True
else:
task.depends_on = {0: task.depends_on, "__script": script_node}
task.depends_on = {0: task.depends_on, JULIA_SCRIPT_KEY: script_node}
task.attributes["julia_keep_dict"] = False

parsed_project = _parse_project(project, task.path.parent)

task.function = functools.partial(
task.function,
script=task.depends_on["__script"].path,
script=task.depends_on[JULIA_SCRIPT_KEY].path,
options=options,
project=parsed_project,
)
Expand Down
4 changes: 2 additions & 2 deletions src/pytask_julia/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None:
if config["julia_serializer"] not in SERIALIZERS:
raise ValueError(
f"'julia_serializer' is {config['julia_serializer']} and not one of "
f"{list(SERIALIZERS)}."
f"{list(SERIALIZERS)}.",
)
config["julia_suffix"] = config.get("julia_suffix", "")
config["julia_options"] = _parse_value_or_whitespace_option(
config.get("julia_options")
config.get("julia_options"),
)
project = config.get("julia_project")
if project is None:
Expand Down
12 changes: 8 additions & 4 deletions src/pytask_julia/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from pytask_julia.serialization import create_path_to_serialized
from pytask_julia.serialization import serialize_keyword_arguments
from pytask_julia.shared import julia
from pytask_julia.shared import JULIA_SCRIPT_KEY


@hookimpl
Expand All @@ -22,7 +23,7 @@ def pytask_execute_task_setup(task: Task) -> None:
if shutil.which("julia") is None:
raise RuntimeError(
"julia is needed to run Julia scripts, but it is not found on your "
"PATH."
"PATH.",
)

assert len(marks) == 1
Expand All @@ -45,13 +46,16 @@ def collect_keyword_arguments(task: Task) -> dict[str, Any]:
kwargs = dict(task.kwargs)
task.kwargs = {}

if len(task.depends_on) == 1 and "__script" in task.depends_on:
if len(task.depends_on) == 1 and JULIA_SCRIPT_KEY in task.depends_on:
pass
elif not task.attributes["julia_keep_dict"] and len(task.depends_on) == 2:
elif (
not task.attributes["julia_keep_dict"]
and len(task.depends_on) == 2 # noqa: PLR2004
):
kwargs["depends_on"] = str(task.depends_on[0].value)
else:
kwargs["depends_on"] = tree_map(lambda x: str(x.value), task.depends_on)
kwargs["depends_on"].pop("__script")
kwargs["depends_on"].pop(JULIA_SCRIPT_KEY)

if task.produces:
kwargs["produces"] = tree_map(lambda x: str(x.value), task.produces)
Expand Down
2 changes: 1 addition & 1 deletion src/pytask_julia/parametrize.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@
@hookimpl
def pytask_parametrize_kwarg_to_marker(obj: Any, kwargs: dict[str, Any]) -> None:
"""Attach parametrized Julia arguments to the function with a marker."""
if callable(obj) and "julia" in kwargs:
if callable(obj) and "julia" in kwargs: # noqa: PLR2004
pytask.mark.julia(**kwargs.pop("julia"))(obj)
3 changes: 3 additions & 0 deletions src/pytask_julia/shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
from typing import Sequence


JULIA_SCRIPT_KEY = "__script"


def julia(
script: str | Path,
options: str | Iterable[str] | None = None,
Expand Down
3 changes: 2 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@


needs_julia = pytest.mark.skipif(
shutil.which("julia") is None, reason="julia needs to be installed."
shutil.which("julia") is None,
reason="julia needs to be installed.",
)


Expand Down
6 changes: 5 additions & 1 deletion tests/test_collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,11 @@ def test_parse_julia_mark(
):
with expectation:
out = _parse_julia_mark(
mark, default_options, default_serializer, default_suffix, default_project
mark,
default_options,
default_serializer,
default_suffix,
default_project,
)
assert out == expected

Expand Down
67 changes: 54 additions & 13 deletions tests/test_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,14 @@ def test_pytask_execute_task_setup_missing_julia(monkeypatch):
"""Make sure that the task setup raises errors."""
# Act like julia is installed since we do not test this.
monkeypatch.setattr(
"pytask_julia.execute.shutil.which", lambda x: None # noqa: ARG005
"pytask_julia.execute.shutil.which",
lambda x: None, # noqa: ARG005
)
task = Task(
base_name="example", path=Path(), function=None, markers=[Mark("julia", (), {})]
base_name="example",
path=Path(),
function=None,
markers=[Mark("julia", (), {})],
)
with pytest.raises(RuntimeError, match="julia is needed"):
pytask_execute_task_setup(task)
Expand All @@ -37,7 +41,12 @@ def test_pytask_execute_task_setup_missing_julia(monkeypatch):
@parametrize_parse_code_serializer_suffix
@pytest.mark.parametrize("depends_on", ["'in_1.txt'", "['in_1.txt', 'in_2.txt']"])
def test_run_jl_script(
runner, tmp_path, parse_config_code, serializer, suffix, depends_on
runner,
tmp_path,
parse_config_code,
serializer,
suffix,
depends_on,
):
task_source = f"""
import pytask
Expand Down Expand Up @@ -75,15 +84,20 @@ def task_run_jl_script():
assert result.exit_code == ExitCode.OK
assert tmp_path.joinpath("out.txt").exists()
assert tmp_path.joinpath(
".pytask", "task_dummy_py_task_run_jl_script" + suffix
".pytask",
"task_dummy_py_task_run_jl_script" + suffix,
).exists()


@needs_julia
@pytest.mark.end_to_end()
@parametrize_parse_code_serializer_suffix
def test_run_jl_script_w_task_decorator(
runner, tmp_path, parse_config_code, serializer, suffix
runner,
tmp_path,
parse_config_code,
serializer,
suffix,
):
task_source = f"""
import pytask
Expand Down Expand Up @@ -118,7 +132,11 @@ def run_jl_script():
@pytest.mark.end_to_end()
@parametrize_parse_code_serializer_suffix
def test_raise_error_if_julia_is_not_found(
tmp_path, monkeypatch, parse_config_code, serializer, suffix
tmp_path,
monkeypatch,
parse_config_code,
serializer,
suffix,
):
task_source = f"""
import pytask
Expand Down Expand Up @@ -146,7 +164,8 @@ def task_run_jl_script():

# Hide julia if available.
monkeypatch.setattr(
"pytask_julia.execute.shutil.which", lambda x: None # noqa: ARG005
"pytask_julia.execute.shutil.which",
lambda x: None, # noqa: ARG005
)

session = main({"paths": tmp_path})
Expand All @@ -159,7 +178,11 @@ def task_run_jl_script():
@pytest.mark.end_to_end()
@parametrize_parse_code_serializer_suffix
def test_run_jl_script_w_wrong_cmd_option(
runner, tmp_path, parse_config_code, serializer, suffix
runner,
tmp_path,
parse_config_code,
serializer,
suffix,
):
task_source = f"""
import pytask
Expand Down Expand Up @@ -195,7 +218,12 @@ def task_run_jl_script():
@pytest.mark.parametrize("n_threads", [2, 3])
@parametrize_parse_code_serializer_suffix
def test_check_passing_cmd_line_options(
runner, tmp_path, n_threads, parse_config_code, serializer, suffix
runner,
tmp_path,
n_threads,
parse_config_code,
serializer,
suffix,
):
task_source = f"""
import pytask
Expand Down Expand Up @@ -242,7 +270,14 @@ def task_run_jl_script():
)
@pytest.mark.parametrize("path", [ROOT, "relative_from_config"])
def test_run_jl_script_w_environment_in_config(
runner, tmp_path, parse_config_code, serializer, suffix, config_path, value, path
runner,
tmp_path,
parse_config_code,
serializer,
suffix,
config_path,
value,
path,
):
task_source = f"""
import pytask
Expand Down Expand Up @@ -279,7 +314,8 @@ def task_run_jl_script():
assert result.exit_code == ExitCode.OK
assert tmp_path.joinpath("out.txt").exists()
assert tmp_path.joinpath(
".pytask", "task_dummy_py_task_run_jl_script" + suffix
".pytask",
"task_dummy_py_task_run_jl_script" + suffix,
).exists()


Expand All @@ -291,7 +327,11 @@ def task_run_jl_script():
)
@parametrize_parse_code_serializer_suffix
def test_run_jl_script_w_environment_relative_to_task(
runner, tmp_path, parse_config_code, serializer, suffix
runner,
tmp_path,
parse_config_code,
serializer,
suffix,
):
project_in_task = Path(os.path.relpath(ROOT, tmp_path)).as_posix()

Expand Down Expand Up @@ -324,7 +364,8 @@ def task_run_jl_script():
assert result.exit_code == ExitCode.OK
assert tmp_path.joinpath("out.txt").exists()
assert tmp_path.joinpath(
".pytask", "task_dummy_py_task_run_jl_script" + suffix
".pytask",
"task_dummy_py_task_run_jl_script" + suffix,
).exists()


Expand Down
5 changes: 4 additions & 1 deletion tests/test_normal_execution_w_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
)
@pytest.mark.parametrize("products", [("out.txt",), ("out_1.txt", "out_2.txt")])
def test_execution_w_varying_dependencies_products(
runner, tmp_path, dependencies, products
runner,
tmp_path,
dependencies,
products,
):
source = f"""
import pytask
Expand Down
Loading