From e7bca837f08345980f240f633921dbcce174345e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Jul 2024 15:58:25 +0200 Subject: [PATCH] fix mulled v2 hash generation from strings the build information that may be contained in requirement versions is considered as part of the version in some places of the code - mulled container resolvers - mulled-build-tool - [planemo](https://github.com/galaxyproject/planemo/blob/b3e12a334e3d358ce30bf2c8c3f8d5a7a7e08136/planemo/conda.py#L116) in others it isn't: - mulled-hash - mulled-build-files - mulled_build which are the files that use the `target_str_to_targets` function. With this change the build info in version strings is always considered as part of the version. `mulled-build-files` is used in the multi-package-containers repo which led to wrong hashes in up to approx 50 tools (check with `grep "=[^,]*=" combinations/* | wc -l`). --- .../tool_util/deps/mulled/mulled_build.py | 11 ++++----- .../deps/mulled/mulled_build_tool.py | 24 ++++++++++++++----- .../tool_util/deps/mulled/mulled_hash.py | 20 ++++++++++++---- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index be956c84a8b9..fb789075d4b3 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -352,7 +352,7 @@ def mull_targets( with PrintProgress(): ret = involucro_context.exec_command(involucro_args) if singularity: - # we can not remove this folder as it contains the image wich is owned by root + # we can not remove this folder as it contains the image which is owned by root pass # shutil.rmtree('./singularity_import') return ret @@ -531,12 +531,7 @@ def target_str_to_targets(targets_raw): def parse_target(target_str): if "=" in target_str: package_name, version = target_str.split("=", 1) - build = None - if "=" in version: - version, build = version.split("=") - elif "--" in version: - version, build = version.split("--") - target = build_target(package_name, version, build) + target = build_target(package_name, version) else: target = build_target(target_str) return target @@ -584,6 +579,8 @@ def args_to_mull_targets_kwds(args): kwds["singularity_image_dir"] = args.singularity_image_dir if hasattr(args, "invfile"): kwds["invfile"] = args.invfile + if hasattr(args, "base_image"): + kwds["base_image"] = args.base_image kwds["involucro_context"] = context_from_args(args) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py index a469bcd5eac9..bafaf61baa64 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py @@ -27,6 +27,23 @@ from galaxy.tool_util.deps.conda_util import CondaTarget +def _mulled_build_tool(tool, args): + """ + >>> import doctest + >>> doctest.ELLIPSIS_MARKER = '-ignore-' + >>> import argparse + >>> _mulled_build_tool("test/functional/tools/mulled_example_multi_1.xml", argparse.Namespace(dry_run=True, base_image="does-not-matter-here-but-test-is-fast", command="build", verbose=True, involucro_path="./involucro")) # doctest: +ELLIPSIS + -ignore- REPO=quay.io/biocontainers/mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:a6419f25efff953fc505dbd5ee734856180bb619-0 -ignore- + >>> _mulled_build_tool("test/functional/tools/mulled_example_multi_2.xml", argparse.Namespace(dry_run=True, base_image="does-not-matter-here-but-test-is-fast", command="build", verbose=True, involucro_path="./involucro")) # doctest: +ELLIPSIS + -ignore- REPO=quay.io/biocontainers/mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:8e86df67d257ce6494ae12b2c60e1b94025ea529-0 -ignore- + """ + tool_source = get_tool_source(tool) + requirements, *_ = tool_source.parse_requirements_and_containers() + targets = requirements_to_mulled_targets(requirements) + kwds = args_to_mull_targets_kwds(args) + mull_targets(targets, **kwds) + + def main(argv=None) -> None: """Main entry-point for the CLI tool.""" parser = arg_parser(argv, globals()) @@ -35,12 +52,7 @@ def main(argv=None) -> None: parser.add_argument("command", metavar="COMMAND", help="Command (build-and-test, build, all)") parser.add_argument("tool", metavar="TOOL", default=None, help="Path to tool to build mulled image for.") args = parser.parse_args() - tool_source = get_tool_source(args.tool) - requirements, *_ = tool_source.parse_requirements_and_containers() - targets = requirements_to_mulled_targets(requirements) - kwds = args_to_mull_targets_kwds(args) - mull_targets(targets, **kwds) - + _mulled_build_tool(args.tool, args) def requirements_to_mulled_targets(requirements) -> List["CondaTarget"]: """Convert Galaxy's representation of requirements into a list of CondaTarget objects. diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_hash.py b/lib/galaxy/tool_util/deps/mulled/mulled_hash.py index 691e4ab7e749..bd93cfbe1f6b 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_hash.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_hash.py @@ -16,17 +16,29 @@ ) +def _mulled_hash(hash, targets): + """ + >>> _mulled_hash(hash="v2", targets="samtools=1.3.1,bedtools=2.26.0") + 'mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:a6419f25efff953fc505dbd5ee734856180bb619' + >>> _mulled_hash(hash="v2", targets="samtools=1.3.1=h9071d68_10,bedtools=2.26.0=0") + 'mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:8e86df67d257ce6494ae12b2c60e1b94025ea529' + """ + targets = target_str_to_targets(targets) + image_name = v2_image_name if hash == "v2" else v1_image_name + return image_name(targets) + + def main(argv=None): - """Main entry-point for the CLI tool.""" + """ + Main entry-point for the CLI tool. + """ parser = arg_parser(argv, globals()) parser.add_argument( "targets", metavar="TARGETS", default=None, help="Comma-separated packages for calculating the mulled hash." ) parser.add_argument("--hash", dest="hash", choices=["v1", "v2"], default="v2") args = parser.parse_args() - targets = target_str_to_targets(args.targets) - image_name = v2_image_name if args.hash == "v2" else v1_image_name - print(image_name(targets)) + print(_mulled_hash(args.hash, args.targets)) __all__ = ("main",)