From c5c190164ee72ea28c516b5861c5a0ab1abb6956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 19 Apr 2022 17:04:40 +0200 Subject: [PATCH 01/31] don't updatate sha if the module is not updated --- nf_core/modules/update.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 1967ac55a4..f2011f0d66 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -428,12 +428,6 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) - # Don't save to a file, just iteratively update the variable - else: - modules_json = self.update_modules_json( - modules_json, modules_repo.name, module, version, write_file=False - ) - if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From f26c2d4fd3cf42a4fed86d1711ab2ba1f88b79b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 20 Apr 2022 10:16:30 +0200 Subject: [PATCH 02/31] modify changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc9e1067d..2396490ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Bumped the minimum version of `rich` from `v10` to `v10.7.0` - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. +- Fix bug in modules.json when a module is not updated ([#1518](https://github.com/nf-core/tools/pull/1518)) ### Modules From 237475634eab661bc33cf9bbf7c840de63349c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 19 Apr 2022 17:04:40 +0200 Subject: [PATCH 03/31] don't updatate sha if the module is not updated --- nf_core/modules/update.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 1967ac55a4..f2011f0d66 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -428,12 +428,6 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) - # Don't save to a file, just iteratively update the variable - else: - modules_json = self.update_modules_json( - modules_json, modules_repo.name, module, version, write_file=False - ) - if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From 5b04b2b54191b256ac561f5c6e65add39816b289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 20 Apr 2022 10:16:30 +0200 Subject: [PATCH 04/31] resolve conflicts in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 259bd35e28..5298f0a411 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) - Bumped the minimum version of `rich` from `v10` to `v10.7.0` +- Fix bug in modules.json when a module is not updated ([#1518](https://github.com/nf-core/tools/pull/1518)) ### Modules From e6b1af5d151831c158a74a58589806dfb9101b66 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 May 2022 15:25:50 +0200 Subject: [PATCH 05/31] resolve conflicts in main_nf.py --- nf_core/__main__.py | 7 +++-- nf_core/modules/lint/__init__.py | 30 ++++++++++++++------ nf_core/modules/lint/main_nf.py | 48 ++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0c010ae9ab..bd8f9a19c4 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -594,7 +594,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("-a", "--all", is_flag=True, help="Run on all modules") @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") -def lint(ctx, tool, dir, key, all, local, passed): +@click.option("--fix", is_flag=True, help="Fix the module version if a newer version is available") +def lint(ctx, tool, dir, key, all, local, passed, fix): """ Lint one or more modules in a directory. @@ -607,7 +608,9 @@ def lint(ctx, tool, dir, key, all, local, passed): try: module_lint = nf_core.modules.ModuleLint(dir=dir) module_lint.modules_repo = ctx.obj["modules_repo_obj"] - module_lint.lint(module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed) + module_lint.lint( + module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed, fix_version=fix + ) if len(module_lint.failed) > 0: sys.exit(1) except nf_core.modules.lint.ModuleLintException as e: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 6bea05cb27..91a9455899 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -100,7 +100,16 @@ def __init__(self, dir): def _get_all_lint_tests(): return ["main_nf", "meta_yml", "module_todos", "module_deprecations"] - def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False): + def lint( + self, + module=None, + key=(), + all_modules=False, + print_results=True, + show_passed=False, + local=False, + fix_version=False, + ): """ Lint all or one specific module @@ -117,6 +126,7 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ :param module: A specific module to lint :param print_results: Whether to print the linting results :param show_passed: Whether passed tests should be shown as well + :param fix_version: Update the module version if a newer version is available :returns: A ModuleLint object containing information of the passed, warned and failed tests @@ -173,11 +183,11 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ # Lint local modules if local and len(local_modules) > 0: - self.lint_modules(local_modules, local=True) + self.lint_modules(local_modules, local=True, fix_version=fix_version) # Lint nf-core modules if len(nfcore_modules) > 0: - self.lint_modules(nfcore_modules, local=False) + self.lint_modules(nfcore_modules, local=False, fix_version=fix_version) if print_results: self._print_results(show_passed=show_passed) @@ -280,13 +290,14 @@ def get_installed_modules(self): return local_modules, nfcore_modules - def lint_modules(self, modules, local=False): + def lint_modules(self, modules, local=False, fix_version=False): """ Lint a list of modules Args: modules ([NFCoreModule]): A list of module objects local (boolean): Whether the list consist of local or nf-core modules + fix_version (boolean): Fix the module version if a newer version is available """ progress_bar = rich.progress.Progress( "[bold blue]{task.description}", @@ -303,9 +314,9 @@ def lint_modules(self, modules, local=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, local=local) + self.lint_module(mod, local=local, fix_version=fix_version) - def lint_module(self, mod, local=False): + def lint_module(self, mod, local=False, fix_version=False): """ Perform linting on one module @@ -324,14 +335,17 @@ def lint_module(self, mod, local=False): # Only check the main script in case of a local module if local: - self.main_nf(mod) + self.main_nf(mod, fix_version) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] # Otherwise run all the lint tests else: for test_name in self.lint_tests: - getattr(self, test_name)(mod) + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version) + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in mod.warned] diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 31dc6d5495..0d30629b77 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -6,8 +6,10 @@ import re import nf_core +from galaxy.tool_util.deps.mulled.util import build_target -def main_nf(module_lint_object, module): + +def main_nf(module_lint_object, module, fix_version): """ Lint a ``main.nf`` module file @@ -99,7 +101,7 @@ def main_nf(module_lint_object, module): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines): + if check_process_section(module, process_lines, fix_version): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -189,7 +191,7 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines): +def check_process_section(self, lines, fix_version): """ Lint the section of a module between the process definition and the 'input:' definition @@ -208,6 +210,7 @@ def check_process_section(self, lines): singularity_tag = "singularity" docker_tag = "docker" bioconda_packages = [] + update = False # Process name should be all capital letters self.process_name = lines[0].split()[1] @@ -235,14 +238,14 @@ def check_process_section(self, lines): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for l in lines: - if re.search("bioconda::", l): + if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] l = l.strip(" '\"") - if l.startswith("https://containers") or l.startswith("https://depot"): + if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 singularity_tag = re.search("(?:\/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_\.]+)(?:\.img)?['\"]", l).group(1) - if l.startswith("biocontainers/") or l.startswith("quay.io/"): + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 docker_tag = re.search("(?:[\/])?(?::)?([A-Za-z\d\-_\.]+)['\"]", l).group(1) @@ -274,10 +277,15 @@ def check_process_section(self, lines): self.warned.append( ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) ) + update = True else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) + print(docker_tag, singularity_tag) if docker_tag == singularity_tag: + # If linting was successful and a new version is available and fix is True + if fix_version and update: + _fix_module_version(self, bioconda_version, last_ver, singularity_tag) return True else: return False @@ -340,3 +348,31 @@ def _is_empty(self, line): if line.strip().replace(" ", "") == "": empty = True return empty + + +def _fix_module_version(self, current_version, latest_version, singularity_tag): + """Updates the module version""" + # Get target object from the latest version + target = build_target(self.module_name, latest_version) + + with open(module.main_nf, "r") as source: + lines = source.readlines() + # Replace outdated versions by the latest one + with open(module.main_nf, "w") as source: + for line in lines: + line = line.strip(" '\"") + build_type = _container_type(line) + if build_type == "bioconda": + source.write(re.sub(rf"{current_version}", f"{latest_version}", line)) + elif build_type == "singularity" or build_type == "docker": + source.write(re.sub(rf"{singularity_tag}", f"{target.build}", line)) + + +def _container_type(line): + """Returns the container type of a build.""" + if re.search("bioconda::", line): + return "bioconda" + if line.startswith("https://containers") or line.startswith("https://depot"): + return "singularity" + if line.startswith("biocontainers/") or line.startswith("quay.io/"): + return "docker" From cee29f03ad549792f0389100ceb5507fd6420309 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 08:56:13 +0200 Subject: [PATCH 06/31] fix conflicts --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5298f0a411..259bd35e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,6 @@ - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) - Bumped the minimum version of `rich` from `v10` to `v10.7.0` -- Fix bug in modules.json when a module is not updated ([#1518](https://github.com/nf-core/tools/pull/1518)) ### Modules From 21d9a546b7beaf47e324eb1d770c785d220e797f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 08:51:12 +0200 Subject: [PATCH 07/31] get version and build --- nf_core/modules/lint/main_nf.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0d30629b77..c4924cea0c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -7,6 +7,7 @@ import nf_core from galaxy.tool_util.deps.mulled.util import build_target +import nf_core.modules.module_utils def main_nf(module_lint_object, module, fix_version): @@ -281,11 +282,10 @@ def check_process_section(self, lines, fix_version): else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) - print(docker_tag, singularity_tag) if docker_tag == singularity_tag: # If linting was successful and a new version is available and fix is True if fix_version and update: - _fix_module_version(self, bioconda_version, last_ver, singularity_tag) + _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response) return True else: return False @@ -350,22 +350,37 @@ def _is_empty(self, line): return empty -def _fix_module_version(self, current_version, latest_version, singularity_tag): +def _fix_module_version(self, current_version, latest_version, singularity_tag, response): """Updates the module version""" # Get target object from the latest version - target = build_target(self.module_name, latest_version) + build = _get_build(response) - with open(module.main_nf, "r") as source: + with open(self.main_nf, "r") as source: lines = source.readlines() # Replace outdated versions by the latest one - with open(module.main_nf, "w") as source: + with open(self.main_nf, "w") as source: for line in lines: - line = line.strip(" '\"") - build_type = _container_type(line) + l = line.strip(" '\"") + build_type = _container_type(l) if build_type == "bioconda": source.write(re.sub(rf"{current_version}", f"{latest_version}", line)) elif build_type == "singularity" or build_type == "docker": - source.write(re.sub(rf"{singularity_tag}", f"{target.build}", line)) + source.write(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) + else: + source.write(line) + ### CHECK URLS BEFORE WRITING + ### MOVE FUNCTION CALL UPPER TO WRITE LOG & DEBUG MESSAGES + + +def _get_build(response): + """Get the build of the container version""" + build_times = [] + latest_v = response.get("latest_version") + files = response.get("files") + for f in files: + if f.get("version") == latest_v: + build_times.append((f.get("upload_time"), f.get("attrs").get("build"))) + return sorted(build_times, key=lambda tup: tup[0], reverse=True)[0][1] def _container_type(line): From db7fac1e3b05d1b5adac7ac5501c13604edc6519 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 09:57:05 +0200 Subject: [PATCH 08/31] check that url exist & add log info --- nf_core/modules/lint/main_nf.py | 70 ++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c4924cea0c..9fe6cfc12a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -5,10 +5,14 @@ import re import nf_core +import logging +import requests from galaxy.tool_util.deps.mulled.util import build_target import nf_core.modules.module_utils +log = logging.getLogger(__name__) + def main_nf(module_lint_object, module, fix_version): """ @@ -211,7 +215,6 @@ def check_process_section(self, lines, fix_version): singularity_tag = "singularity" docker_tag = "docker" bioconda_packages = [] - update = False # Process name should be all capital letters self.process_name = lines[0].split()[1] @@ -275,17 +278,31 @@ def check_process_section(self, lines, fix_version): last_ver = response.get("latest_version") if last_ver is not None and last_ver != bioconda_version: package, ver = bp.split("=", 1) - self.warned.append( - ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) - ) - update = True + # If a new version is available and fix is True, update the version + if fix_version: + if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): + log.info(f"Updating package {package} `{ver}` -> `{last_ver}`") + log.debug(f"Updating package {package} `{ver}` -> `{last_ver}`") + self.passed.append( + ( + "bioconda_latest", + f"Conda package has been updated to the latest available: `{bp}`", + self.main_nf, + ) + ) + else: + log.debug(f"Unable to updating package {package} `{ver}` -> `{last_ver}`") + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) + else: + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) if docker_tag == singularity_tag: - # If linting was successful and a new version is available and fix is True - if fix_version and update: - _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response) return True else: return False @@ -357,19 +374,34 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, with open(self.main_nf, "r") as source: lines = source.readlines() + + # Check if the new version + build exist and replace + new_lines = [] + for line in lines: + l = line.strip(" '\"") + build_type = _container_type(l) + if build_type == "bioconda": + new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) + elif build_type == "singularity" or build_type == "docker": + # Check that the new url is valid + new_url = re.search( + "(?:')(.+)(?:')", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line) + ).group(1) + response_new_container = requests.get( + "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True + ) + if response_new_container.status_code != 200: + return False + new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) + else: + new_lines.append(line) + # Replace outdated versions by the latest one with open(self.main_nf, "w") as source: - for line in lines: - l = line.strip(" '\"") - build_type = _container_type(l) - if build_type == "bioconda": - source.write(re.sub(rf"{current_version}", f"{latest_version}", line)) - elif build_type == "singularity" or build_type == "docker": - source.write(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) - else: - source.write(line) - ### CHECK URLS BEFORE WRITING - ### MOVE FUNCTION CALL UPPER TO WRITE LOG & DEBUG MESSAGES + for line in new_lines: + source.write(line) + + return True def _get_build(response): From 72c49c917503c482f106e5bc686e260aa2f212d2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 11:33:53 +0200 Subject: [PATCH 09/31] improve logging --- nf_core/modules/lint/__init__.py | 9 +++++---- nf_core/modules/lint/main_nf.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 91a9455899..a588f49255 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -304,6 +304,7 @@ def lint_modules(self, modules, local=False, fix_version=False): rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, + console=console, ) with progress_bar: lint_progress = progress_bar.add_task( @@ -314,9 +315,9 @@ def lint_modules(self, modules, local=False, fix_version=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, local=local, fix_version=fix_version) + self.lint_module(mod, progress_bar, local=local, fix_version=fix_version) - def lint_module(self, mod, local=False, fix_version=False): + def lint_module(self, mod, progress_bar, local=False, fix_version=False): """ Perform linting on one module @@ -335,7 +336,7 @@ def lint_module(self, mod, local=False, fix_version=False): # Only check the main script in case of a local module if local: - self.main_nf(mod, fix_version) + self.main_nf(mod, fix_version, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] @@ -343,7 +344,7 @@ def lint_module(self, mod, local=False, fix_version=False): else: for test_name in self.lint_tests: if test_name == "main_nf": - getattr(self, test_name)(mod, fix_version) + getattr(self, test_name)(mod, fix_version, progress_bar) else: getattr(self, test_name)(mod) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 9fe6cfc12a..bc30718f8c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -14,7 +14,7 @@ log = logging.getLogger(__name__) -def main_nf(module_lint_object, module, fix_version): +def main_nf(module_lint_object, module, fix_version, progress_bar): """ Lint a ``main.nf`` module file @@ -106,7 +106,7 @@ def main_nf(module_lint_object, module, fix_version): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines, fix_version): + if check_process_section(module, process_lines, fix_version, progress_bar): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -196,7 +196,7 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines, fix_version): +def check_process_section(self, lines, fix_version, progress_bar): """ Lint the section of a module between the process definition and the 'input:' definition @@ -281,7 +281,7 @@ def check_process_section(self, lines, fix_version): # If a new version is available and fix is True, update the version if fix_version: if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): - log.info(f"Updating package {package} `{ver}` -> `{last_ver}`") + progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}") log.debug(f"Updating package {package} `{ver}` -> `{last_ver}`") self.passed.append( ( @@ -291,7 +291,7 @@ def check_process_section(self, lines, fix_version): ) ) else: - log.debug(f"Unable to updating package {package} `{ver}` -> `{last_ver}`") + log.debug(f"Unable to update package {package} `{ver}` -> `{last_ver}`") self.warned.append( ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) ) From 008350ededcc7c565b709e8a51d563e1b6eae6ee Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 11:42:04 +0200 Subject: [PATCH 10/31] improve comments --- nf_core/modules/lint/main_nf.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index bc30718f8c..b170a3b6cd 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -368,8 +368,14 @@ def _is_empty(self, line): def _fix_module_version(self, current_version, latest_version, singularity_tag, response): - """Updates the module version""" - # Get target object from the latest version + """Updates the module version + + Changes the bioconda current version by the latest version. + Obtains the latest build from bioconda response + Checks that the new URLs for docker and singularity with the tag [version]--[build] are valid + Changes the docker and singularity URLs + """ + # Get latest build build = _get_build(response) with open(self.main_nf, "r") as source: @@ -405,7 +411,7 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, def _get_build(response): - """Get the build of the container version""" + """Get the latest build of the container version""" build_times = [] latest_v = response.get("latest_version") files = response.get("files") From e152b7be98e63634874f4dc2a060fd5596aa8bb4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 12:49:12 +0200 Subject: [PATCH 11/31] remove changes made by error --- nf_core/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index f2011f0d66..1967ac55a4 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -428,6 +428,12 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) + # Don't save to a file, just iteratively update the variable + else: + modules_json = self.update_modules_json( + modules_json, modules_repo.name, module, version, write_file=False + ) + if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From 106b60e7534f186b8847b18f02e35f1966a6e885 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 14:24:03 +0200 Subject: [PATCH 12/31] update changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6084931ff2..ecee8b46ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # nf-core/tools: Changelog +## v2.4.2dev + +- Add `fix` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) + ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] - Patch release to try to fix the template sync ([#1585](https://github.com/nf-core/tools/pull/1585)) @@ -30,7 +34,6 @@ - Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) - Bumped the minimum version of `rich` from `v10` to `v10.7.0` - ### Modules - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. From c1c4541058a7795d2518d5fa41ebdde8e88bf89e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 14:44:26 +0200 Subject: [PATCH 13/31] small changelog change --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe78f9993e..d6699bf2e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,5 @@ # nf-core/tools: Changelog - ## v2.5dev ### Template From e6525ef3ff30b7cf59bfdb6816ac241c457f6c65 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 May 2022 14:22:34 +0200 Subject: [PATCH 14/31] improve debug messages --- nf_core/modules/lint/main_nf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index b170a3b6cd..d1c568f4f1 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -282,7 +282,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if fix_version: if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}") - log.debug(f"Updating package {package} `{ver}` -> `{last_ver}`") + log.debug(f"Updating package {package} {ver} -> {last_ver}") self.passed.append( ( "bioconda_latest", @@ -291,7 +291,10 @@ def check_process_section(self, lines, fix_version, progress_bar): ) ) else: - log.debug(f"Unable to update package {package} `{ver}` -> `{last_ver}`") + progress_bar.print( + f"[blue]INFO[/blue]\t Tried to update package. Unable to update package '{package}' {ver} -> {last_ver}" + ) + log.debug(f"Unable to update package {package} {ver} -> {last_ver}") self.warned.append( ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) ) @@ -396,6 +399,9 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, response_new_container = requests.get( "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True ) + log.debug( + f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}" + ) if response_new_container.status_code != 200: return False new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) From 0122056fd7d2b365db9b66528ca6b4d3e146a08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 19 Apr 2022 17:04:40 +0200 Subject: [PATCH 15/31] don't updatate sha if the module is not updated --- nf_core/modules/update.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 150e787a74..ad68cb6c28 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -433,12 +433,6 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) - # Don't save to a file, just iteratively update the variable - else: - modules_json = self.update_modules_json( - modules_json, modules_repo.name, module, version, write_file=False - ) - if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From b288578a00f65a6b64f750a54d62675d4b3a63d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 20 Apr 2022 10:16:30 +0200 Subject: [PATCH 16/31] resolve conflicts in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a287038b4..e6ae3b38b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) - Bumped the minimum version of `rich` from `v10` to `v10.7.0` +- Fix bug in modules.json when a module is not updated ([#1518](https://github.com/nf-core/tools/pull/1518)) ### Modules From 7b1538ea947b7cfc3b19cbd226df83df0c62d52c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 May 2022 15:25:50 +0200 Subject: [PATCH 17/31] resolve conflicts in main_nf.py --- nf_core/__main__.py | 7 +++-- nf_core/modules/lint/__init__.py | 30 ++++++++++++++------ nf_core/modules/lint/main_nf.py | 48 ++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 50a65f4778..38b70371b9 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -596,7 +596,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("-a", "--all", is_flag=True, help="Run on all modules") @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") -def lint(ctx, tool, dir, key, all, local, passed): +@click.option("--fix", is_flag=True, help="Fix the module version if a newer version is available") +def lint(ctx, tool, dir, key, all, local, passed, fix): """ Lint one or more modules in a directory. @@ -609,7 +610,9 @@ def lint(ctx, tool, dir, key, all, local, passed): try: module_lint = nf_core.modules.ModuleLint(dir=dir) module_lint.modules_repo = ctx.obj["modules_repo_obj"] - module_lint.lint(module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed) + module_lint.lint( + module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed, fix_version=fix + ) if len(module_lint.failed) > 0: sys.exit(1) except nf_core.modules.lint.ModuleLintException as e: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index ed9db99371..407eb73416 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -101,7 +101,16 @@ def __init__(self, dir): def _get_all_lint_tests(): return ["main_nf", "meta_yml", "module_todos", "module_deprecations"] - def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False): + def lint( + self, + module=None, + key=(), + all_modules=False, + print_results=True, + show_passed=False, + local=False, + fix_version=False, + ): """ Lint all or one specific module @@ -118,6 +127,7 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ :param module: A specific module to lint :param print_results: Whether to print the linting results :param show_passed: Whether passed tests should be shown as well + :param fix_version: Update the module version if a newer version is available :returns: A ModuleLint object containing information of the passed, warned and failed tests @@ -174,11 +184,11 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ # Lint local modules if local and len(local_modules) > 0: - self.lint_modules(local_modules, local=True) + self.lint_modules(local_modules, local=True, fix_version=fix_version) # Lint nf-core modules if len(nfcore_modules) > 0: - self.lint_modules(nfcore_modules, local=False) + self.lint_modules(nfcore_modules, local=False, fix_version=fix_version) if print_results: self._print_results(show_passed=show_passed) @@ -282,13 +292,14 @@ def get_installed_modules(self): return local_modules, nfcore_modules - def lint_modules(self, modules, local=False): + def lint_modules(self, modules, local=False, fix_version=False): """ Lint a list of modules Args: modules ([NFCoreModule]): A list of module objects local (boolean): Whether the list consist of local or nf-core modules + fix_version (boolean): Fix the module version if a newer version is available """ progress_bar = rich.progress.Progress( "[bold blue]{task.description}", @@ -305,9 +316,9 @@ def lint_modules(self, modules, local=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, local=local) + self.lint_module(mod, local=local, fix_version=fix_version) - def lint_module(self, mod, local=False): + def lint_module(self, mod, local=False, fix_version=False): """ Perform linting on one module @@ -326,14 +337,17 @@ def lint_module(self, mod, local=False): # Only check the main script in case of a local module if local: - self.main_nf(mod) + self.main_nf(mod, fix_version) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] # Otherwise run all the lint tests else: for test_name in self.lint_tests: - getattr(self, test_name)(mod) + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version) + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in mod.warned] diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 8d745cd186..030918c79a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -5,10 +5,12 @@ import re +from galaxy.tool_util.deps.mulled.util import build_target + import nf_core -def main_nf(module_lint_object, module): +def main_nf(module_lint_object, module, fix_version): """ Lint a ``main.nf`` module file @@ -100,7 +102,7 @@ def main_nf(module_lint_object, module): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines): + if check_process_section(module, process_lines, fix_version): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -190,7 +192,7 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines): +def check_process_section(self, lines, fix_version): """ Lint the section of a module between the process definition and the 'input:' definition @@ -209,6 +211,7 @@ def check_process_section(self, lines): singularity_tag = "singularity" docker_tag = "docker" bioconda_packages = [] + update = False # Process name should be all capital letters self.process_name = lines[0].split()[1] @@ -236,14 +239,14 @@ def check_process_section(self, lines): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for l in lines: - if re.search("bioconda::", l): + if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] l = l.strip(" '\"") - if l.startswith("https://containers") or l.startswith("https://depot"): + if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 singularity_tag = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l).group(1) - if l.startswith("biocontainers/") or l.startswith("quay.io/"): + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 docker_tag = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l).group(1) @@ -275,10 +278,15 @@ def check_process_section(self, lines): self.warned.append( ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) ) + update = True else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) + print(docker_tag, singularity_tag) if docker_tag == singularity_tag: + # If linting was successful and a new version is available and fix is True + if fix_version and update: + _fix_module_version(self, bioconda_version, last_ver, singularity_tag) return True else: return False @@ -341,3 +349,31 @@ def _is_empty(self, line): if line.strip().replace(" ", "") == "": empty = True return empty + + +def _fix_module_version(self, current_version, latest_version, singularity_tag): + """Updates the module version""" + # Get target object from the latest version + target = build_target(self.module_name, latest_version) + + with open(module.main_nf, "r") as source: + lines = source.readlines() + # Replace outdated versions by the latest one + with open(module.main_nf, "w") as source: + for line in lines: + line = line.strip(" '\"") + build_type = _container_type(line) + if build_type == "bioconda": + source.write(re.sub(rf"{current_version}", f"{latest_version}", line)) + elif build_type == "singularity" or build_type == "docker": + source.write(re.sub(rf"{singularity_tag}", f"{target.build}", line)) + + +def _container_type(line): + """Returns the container type of a build.""" + if re.search("bioconda::", line): + return "bioconda" + if line.startswith("https://containers") or line.startswith("https://depot"): + return "singularity" + if line.startswith("biocontainers/") or line.startswith("quay.io/"): + return "docker" From ef00edee5af8bf6750b376f853695b5b94429abe Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 08:56:13 +0200 Subject: [PATCH 18/31] fix conflicts --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6ae3b38b9..5a287038b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,6 @@ - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) - Bumped the minimum version of `rich` from `v10` to `v10.7.0` -- Fix bug in modules.json when a module is not updated ([#1518](https://github.com/nf-core/tools/pull/1518)) ### Modules From f205b1527feb11a1f67b241aec6fd63f005635d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 08:51:12 +0200 Subject: [PATCH 19/31] get version and build --- nf_core/modules/lint/main_nf.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 030918c79a..d9ef144233 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -6,6 +6,7 @@ import re from galaxy.tool_util.deps.mulled.util import build_target +import nf_core.modules.module_utils import nf_core @@ -282,11 +283,10 @@ def check_process_section(self, lines, fix_version): else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) - print(docker_tag, singularity_tag) if docker_tag == singularity_tag: # If linting was successful and a new version is available and fix is True if fix_version and update: - _fix_module_version(self, bioconda_version, last_ver, singularity_tag) + _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response) return True else: return False @@ -351,22 +351,37 @@ def _is_empty(self, line): return empty -def _fix_module_version(self, current_version, latest_version, singularity_tag): +def _fix_module_version(self, current_version, latest_version, singularity_tag, response): """Updates the module version""" # Get target object from the latest version - target = build_target(self.module_name, latest_version) + build = _get_build(response) - with open(module.main_nf, "r") as source: + with open(self.main_nf, "r") as source: lines = source.readlines() # Replace outdated versions by the latest one - with open(module.main_nf, "w") as source: + with open(self.main_nf, "w") as source: for line in lines: - line = line.strip(" '\"") - build_type = _container_type(line) + l = line.strip(" '\"") + build_type = _container_type(l) if build_type == "bioconda": source.write(re.sub(rf"{current_version}", f"{latest_version}", line)) elif build_type == "singularity" or build_type == "docker": - source.write(re.sub(rf"{singularity_tag}", f"{target.build}", line)) + source.write(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) + else: + source.write(line) + ### CHECK URLS BEFORE WRITING + ### MOVE FUNCTION CALL UPPER TO WRITE LOG & DEBUG MESSAGES + + +def _get_build(response): + """Get the build of the container version""" + build_times = [] + latest_v = response.get("latest_version") + files = response.get("files") + for f in files: + if f.get("version") == latest_v: + build_times.append((f.get("upload_time"), f.get("attrs").get("build"))) + return sorted(build_times, key=lambda tup: tup[0], reverse=True)[0][1] def _container_type(line): From 877b6f69a2a27dd36242f63818b82769f664b7f7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Jun 2022 10:48:57 +0200 Subject: [PATCH 20/31] resolve rebase conflicts --- nf_core/modules/lint/main_nf.py | 76 ++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d9ef144233..072df19239 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -4,11 +4,21 @@ """ import re +<<<<<<< HEAD +======= +import nf_core +import logging +import requests +>>>>>>> db7fac1e (check that url exist & add log info) from galaxy.tool_util.deps.mulled.util import build_target import nf_core.modules.module_utils +<<<<<<< HEAD import nf_core +======= +log = logging.getLogger(__name__) +>>>>>>> db7fac1e (check that url exist & add log info) def main_nf(module_lint_object, module, fix_version): @@ -212,7 +222,6 @@ def check_process_section(self, lines, fix_version): singularity_tag = "singularity" docker_tag = "docker" bioconda_packages = [] - update = False # Process name should be all capital letters self.process_name = lines[0].split()[1] @@ -276,17 +285,31 @@ def check_process_section(self, lines, fix_version): last_ver = response.get("latest_version") if last_ver is not None and last_ver != bioconda_version: package, ver = bp.split("=", 1) - self.warned.append( - ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) - ) - update = True + # If a new version is available and fix is True, update the version + if fix_version: + if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): + log.info(f"Updating package {package} `{ver}` -> `{last_ver}`") + log.debug(f"Updating package {package} `{ver}` -> `{last_ver}`") + self.passed.append( + ( + "bioconda_latest", + f"Conda package has been updated to the latest available: `{bp}`", + self.main_nf, + ) + ) + else: + log.debug(f"Unable to updating package {package} `{ver}` -> `{last_ver}`") + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) + else: + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) if docker_tag == singularity_tag: - # If linting was successful and a new version is available and fix is True - if fix_version and update: - _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response) return True else: return False @@ -358,19 +381,34 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, with open(self.main_nf, "r") as source: lines = source.readlines() + + # Check if the new version + build exist and replace + new_lines = [] + for line in lines: + l = line.strip(" '\"") + build_type = _container_type(l) + if build_type == "bioconda": + new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) + elif build_type == "singularity" or build_type == "docker": + # Check that the new url is valid + new_url = re.search( + "(?:')(.+)(?:')", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line) + ).group(1) + response_new_container = requests.get( + "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True + ) + if response_new_container.status_code != 200: + return False + new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) + else: + new_lines.append(line) + # Replace outdated versions by the latest one with open(self.main_nf, "w") as source: - for line in lines: - l = line.strip(" '\"") - build_type = _container_type(l) - if build_type == "bioconda": - source.write(re.sub(rf"{current_version}", f"{latest_version}", line)) - elif build_type == "singularity" or build_type == "docker": - source.write(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) - else: - source.write(line) - ### CHECK URLS BEFORE WRITING - ### MOVE FUNCTION CALL UPPER TO WRITE LOG & DEBUG MESSAGES + for line in new_lines: + source.write(line) + + return True def _get_build(response): From 3886ad18d0c83c0c8b9949fbc284aaf95cb7304a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 11:33:53 +0200 Subject: [PATCH 21/31] improve logging --- nf_core/modules/lint/__init__.py | 9 +++++---- nf_core/modules/lint/main_nf.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 407eb73416..9bbe173ef5 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -306,6 +306,7 @@ def lint_modules(self, modules, local=False, fix_version=False): rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, + console=console, ) with progress_bar: lint_progress = progress_bar.add_task( @@ -316,9 +317,9 @@ def lint_modules(self, modules, local=False, fix_version=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, local=local, fix_version=fix_version) + self.lint_module(mod, progress_bar, local=local, fix_version=fix_version) - def lint_module(self, mod, local=False, fix_version=False): + def lint_module(self, mod, progress_bar, local=False, fix_version=False): """ Perform linting on one module @@ -337,7 +338,7 @@ def lint_module(self, mod, local=False, fix_version=False): # Only check the main script in case of a local module if local: - self.main_nf(mod, fix_version) + self.main_nf(mod, fix_version, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] @@ -345,7 +346,7 @@ def lint_module(self, mod, local=False, fix_version=False): else: for test_name in self.lint_tests: if test_name == "main_nf": - getattr(self, test_name)(mod, fix_version) + getattr(self, test_name)(mod, fix_version, progress_bar) else: getattr(self, test_name)(mod) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 072df19239..a28cdef8ca 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -21,7 +21,7 @@ >>>>>>> db7fac1e (check that url exist & add log info) -def main_nf(module_lint_object, module, fix_version): +def main_nf(module_lint_object, module, fix_version, progress_bar): """ Lint a ``main.nf`` module file @@ -113,7 +113,7 @@ def main_nf(module_lint_object, module, fix_version): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines, fix_version): + if check_process_section(module, process_lines, fix_version, progress_bar): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -203,7 +203,7 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines, fix_version): +def check_process_section(self, lines, fix_version, progress_bar): """ Lint the section of a module between the process definition and the 'input:' definition @@ -288,7 +288,7 @@ def check_process_section(self, lines, fix_version): # If a new version is available and fix is True, update the version if fix_version: if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): - log.info(f"Updating package {package} `{ver}` -> `{last_ver}`") + progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}") log.debug(f"Updating package {package} `{ver}` -> `{last_ver}`") self.passed.append( ( @@ -298,7 +298,7 @@ def check_process_section(self, lines, fix_version): ) ) else: - log.debug(f"Unable to updating package {package} `{ver}` -> `{last_ver}`") + log.debug(f"Unable to update package {package} `{ver}` -> `{last_ver}`") self.warned.append( ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) ) From eac514d1368fb99f06ebc153bdf9372ffdb4dd92 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 11:42:04 +0200 Subject: [PATCH 22/31] improve comments --- nf_core/modules/lint/main_nf.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index a28cdef8ca..82c94810e3 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -375,8 +375,14 @@ def _is_empty(self, line): def _fix_module_version(self, current_version, latest_version, singularity_tag, response): - """Updates the module version""" - # Get target object from the latest version + """Updates the module version + + Changes the bioconda current version by the latest version. + Obtains the latest build from bioconda response + Checks that the new URLs for docker and singularity with the tag [version]--[build] are valid + Changes the docker and singularity URLs + """ + # Get latest build build = _get_build(response) with open(self.main_nf, "r") as source: @@ -412,7 +418,7 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, def _get_build(response): - """Get the build of the container version""" + """Get the latest build of the container version""" build_times = [] latest_v = response.get("latest_version") files = response.get("files") From a5ba61207158b54bda7678581abf2a239803745f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 17 May 2022 12:49:12 +0200 Subject: [PATCH 23/31] remove changes made by error --- nf_core/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index ad68cb6c28..150e787a74 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -433,6 +433,12 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) + # Don't save to a file, just iteratively update the variable + else: + modules_json = self.update_modules_json( + modules_json, modules_repo.name, module, version, write_file=False + ) + if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From 08e1b945146f3dfdfff080e954aba6fced22bb13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 19 Apr 2022 17:04:40 +0200 Subject: [PATCH 24/31] don't updatate sha if the module is not updated --- nf_core/modules/update.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 150e787a74..ad68cb6c28 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -433,12 +433,6 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) - # Don't save to a file, just iteratively update the variable - else: - modules_json = self.update_modules_json( - modules_json, modules_repo.name, module, version, write_file=False - ) - if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From 99bf43083ee073fd05a5b345907da68b7239526c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Jun 2022 10:56:05 +0200 Subject: [PATCH 25/31] resolve rebase conflicts --- CHANGELOG.md | 6 ++---- nf_core/modules/lint/main_nf.py | 14 ++++---------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a287038b4..2059c0477a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ### Linting - Check that the `.prettierignore` file exists and that starts with the same content. +- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) +- Add `fix` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) ### General @@ -20,10 +22,6 @@ - Fix a bug in the regex extracting the version from biocontainers URLs [#1598](https://github.com/nf-core/tools/pull/1598) -### Linting - -- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) - ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] - Patch release to try to fix the template sync ([#1585](https://github.com/nf-core/tools/pull/1585)) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 82c94810e3..f4adb8f2ee 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -3,22 +3,16 @@ Lint the main.nf file of a module """ -import re -<<<<<<< HEAD -======= -import nf_core import logging -import requests ->>>>>>> db7fac1e (check that url exist & add log info) +import re +import requests from galaxy.tool_util.deps.mulled.util import build_target -import nf_core.modules.module_utils -<<<<<<< HEAD import nf_core -======= +import nf_core.modules.module_utils + log = logging.getLogger(__name__) ->>>>>>> db7fac1e (check that url exist & add log info) def main_nf(module_lint_object, module, fix_version, progress_bar): From 550f6d2d72aff48bd1c22cb3218ff23055a7beaf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 May 2022 14:22:34 +0200 Subject: [PATCH 26/31] improve debug messages --- nf_core/modules/lint/main_nf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index f4adb8f2ee..44fd4e8353 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -283,7 +283,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if fix_version: if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}") - log.debug(f"Updating package {package} `{ver}` -> `{last_ver}`") + log.debug(f"Updating package {package} {ver} -> {last_ver}") self.passed.append( ( "bioconda_latest", @@ -292,7 +292,10 @@ def check_process_section(self, lines, fix_version, progress_bar): ) ) else: - log.debug(f"Unable to update package {package} `{ver}` -> `{last_ver}`") + progress_bar.print( + f"[blue]INFO[/blue]\t Tried to update package. Unable to update package '{package}' {ver} -> {last_ver}" + ) + log.debug(f"Unable to update package {package} {ver} -> {last_ver}") self.warned.append( ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) ) @@ -397,6 +400,9 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, response_new_container = requests.get( "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True ) + log.debug( + f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}" + ) if response_new_container.status_code != 200: return False new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) From 3e9076c61718f0aeae74cff0691e3e7d4e9d2eb8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Jun 2022 14:16:36 +0200 Subject: [PATCH 27/31] refactor fix flag to fix-version --- nf_core/__main__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 38b70371b9..f3f0701fba 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -596,8 +596,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("-a", "--all", is_flag=True, help="Run on all modules") @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") -@click.option("--fix", is_flag=True, help="Fix the module version if a newer version is available") -def lint(ctx, tool, dir, key, all, local, passed, fix): +@click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") +def lint(ctx, tool, dir, key, all, local, passed, fix_version): """ Lint one or more modules in a directory. @@ -611,7 +611,13 @@ def lint(ctx, tool, dir, key, all, local, passed, fix): module_lint = nf_core.modules.ModuleLint(dir=dir) module_lint.modules_repo = ctx.obj["modules_repo_obj"] module_lint.lint( - module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed, fix_version=fix + module=tool, + key=key, + all_modules=all, + print_results=True, + local=local, + show_passed=passed, + fix_version=fix_version, ) if len(module_lint.failed) > 0: sys.exit(1) From 8e061f1bbc5ee922a7576746b720b92bb402e5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 2 Jun 2022 14:59:42 +0200 Subject: [PATCH 28/31] fix typo in CHANGELOG.md Co-authored-by: Gisela Gabernet --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7421f50e05..842695e7ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ ### Modules -- Add `fix` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) +- Add `--fix-version` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) - Fix a bug in the regex extracting the version from biocontainers URLs [#1598](https://github.com/nf-core/tools/pull/1598) ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] From 0037bf914cf5ea6050d6c651c295e678312c644f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 2 Jun 2022 15:08:03 +0200 Subject: [PATCH 29/31] Remove duplicate in CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 842695e7ec..575d2986e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,6 @@ - Check that the `.prettierignore` file exists and that starts with the same content. - Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) -- Add `fix` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) ### General From cfcf3b6c3ea09a93c011f599712cdd38c3fbdb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 2 Jun 2022 15:08:19 +0200 Subject: [PATCH 30/31] Remove changed made by mistake --- nf_core/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index ad68cb6c28..2ad6afab2d 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -433,6 +433,12 @@ class DiffEnum(enum.Enum): if not dry_run: self.update_modules_json(modules_json, modules_repo.name, module, version) + # Don't save to a file, just iteratively update the variable + else: + modules_json = self.update_modules_json( + modules_json, modules_repo.name, module, version, write_file=False + ) + if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff( From 358bb0a4d56b28372d6394088d34ed41f45277a5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Jun 2022 15:13:11 +0200 Subject: [PATCH 31/31] fix black linting --- nf_core/modules/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 2ad6afab2d..150e787a74 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -438,7 +438,7 @@ class DiffEnum(enum.Enum): modules_json = self.update_modules_json( modules_json, modules_repo.name, module, version, write_file=False ) - + if self.save_diff_fn: # Compare the new modules.json and build a diff modules_json_diff = difflib.unified_diff(