Merge pull request #1752 from mirpedrol/patch

add apply patch reverse when create modules.json

CHANGELOG.md

@@ -74,6 +74,7 @@
 - Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728))
 - Fix broken link in `nf-core modules info`([#1745](https://github.com/nf-core/tools/pull/1745))
 - Fix unbound variable issues and minor refactoring [#1742](https://github.com/nf-core/tools/pull/1742/)
+- Add support for patch when creating `modules.json` file ([#1752](https://github.com/nf-core/tools/pull/1752))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 

nf_core/modules/modules_json.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import shutil
+import tempfile
 from pathlib import Path
 
 import git
@@ -14,6 +15,8 @@
 import nf_core.modules.modules_repo
 import nf_core.utils
 
+from .modules_differ import ModulesDiffer
+
 log = logging.getLogger(__name__)
 
 
@@ -230,7 +233,13 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, base_path, m
  tried_branches = {default_modules_repo.branch}
  found_sha = False
  while True:
- correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
+ # If the module is patched
+ patch_file = module_path / f"{module}.diff"
+ if patch_file.is_file():
+ temp_module_dir = self.try_apply_patch_reverse(module, repo_name, patch_file, module_path)
+ correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo)
+ else:
+ correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
  if correct_commit_sha is None:
  log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.")
  choices = [{"name": "No", "value": None}] + [
@@ -625,6 +634,42 @@ def get_patch_fn(self, module_name, repo_name):
  path = self.modules_json["repos"].get(repo_name, {}).get("modules").get(module_name, {}).get("patch")
  return Path(path) if path is not None else None
 
+ def try_apply_patch_reverse(self, module, repo_name, patch_relpath, module_dir):
+ """
+ Try reverse applying a patch file to the modified module files
+
+ Args:
+ module (str): The name of the module
+ repo_name (str): The name of the repository where the module resides
+ patch_relpath (Path | str): The path to patch file in the pipeline
+ module_dir (Path | str): The module directory in the pipeline
+
+ Returns:
+ (Path | str): The path of the folder where the module patched files are
+
+ Raises:
+ LookupError: If patch was not applied
+ """
+ module_fullname = str(Path(repo_name, module))
+ patch_path = Path(self.dir / patch_relpath)
+
+ try:
+ new_files = ModulesDiffer.try_apply_patch(module, repo_name, patch_path, module_dir, reverse=True)
+ except LookupError as e:
+ raise LookupError(f"Failed to apply patch in reverse for module '{module_fullname}' due to: {e}")
+
+ # Write the patched files to a temporary directory
+ log.debug("Writing patched files to tmpdir")
+ temp_dir = Path(tempfile.mkdtemp())
+ temp_module_dir = temp_dir / module
+ temp_module_dir.mkdir(parents=True, exist_ok=True)
+ for file, new_content in new_files.items():
+ fn = temp_module_dir / file
+ with open(fn, "w") as fh:
+ fh.writelines(new_content)
+
+ return temp_module_dir
+
  def repo_present(self, repo_name):
  """
  Checks if a repo is present in the modules.json file

tests/modules/modules_json.py

@@ -1,6 +1,7 @@
 import json
 import os
 import shutil
+from pathlib import Path
 
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import (
@@ -10,6 +11,7 @@
  NF_CORE_MODULES_REMOTE,
  ModulesRepo,
 )
+from nf_core.modules.patch import ModulePatch
 
 
 def test_get_modules_json(self):
@@ -39,7 +41,7 @@ def test_mod_json_update(self):
 
 
 def test_mod_json_create(self):
- """Test creating a modules.json file from scratch""" ""
+ """Test creating a modules.json file from scratch"""
  mod_json_path = os.path.join(self.pipeline_dir, "modules.json")
  # Remove the existing modules.json file
  os.remove(mod_json_path)
@@ -62,6 +64,52 @@ def test_mod_json_create(self):
  assert "branch" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod]
 
 
+def modify_main_nf(path):
+ """Modify a file to test patch creation"""
+ with open(path, "r") as fh:
+ lines = fh.readlines()
+ # Modify $meta.id to $meta.single_end
+ lines[1] = ' tag "$meta.single_end"\n'
+ with open(path, "w") as fh:
+ fh.writelines(lines)
+
+
+def test_mod_json_create_with_patch(self):
+ """Test creating a modules.json file from scratch when there are patched modules"""
+ mod_json_path = Path(self.pipeline_dir, "modules.json")
+
+ # Modify the module
+ module_path = Path(self.pipeline_dir, "modules", "nf-core", "modules", "fastqc")
+ modify_main_nf(module_path / "main.nf")
+
+ # Try creating a patch file
+ patch_obj = ModulePatch(self.pipeline_dir)
+ patch_obj.patch("fastqc")
+
+ # Remove the existing modules.json file
+ os.remove(mod_json_path)
+
+ # Create the new modules.json file
+ ModulesJson(self.pipeline_dir).create()
+
+ # Check that the file exists
+ assert mod_json_path.is_file()
+
+ # Get the contents of the file
+ mod_json_obj = ModulesJson(self.pipeline_dir)
+ mod_json = mod_json_obj.get_modules_json()
+
+ # Check that fastqc is in the file
+ assert "fastqc" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]
+ assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"]
+ assert "branch" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"]
+
+ # Check that fastqc/main.nf maintains the changes
+ with open(module_path / "main.nf", "r") as fh:
+ lines = fh.readlines()
+ assert lines[1] == ' tag "$meta.single_end"\n'
+
+
 def test_mod_json_up_to_date(self):
  """
  Checks if the modules.json file is up to date

tests/test_modules.py

@@ -132,6 +132,7 @@ def test_modulesrepo_class(self):
  from .modules.modules_json import (
  test_get_modules_json,
  test_mod_json_create,
+ test_mod_json_create_with_patch,
  test_mod_json_dump,
  test_mod_json_get_base_path,
  test_mod_json_get_git_url,