Add support for generic git remotes

CHANGELOG.md

@@ -22,6 +22,7 @@
 
 - Add `--fix-version` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588))
 - Fix a bug in the regex extracting the version from biocontainers URLs ([#1598](https://github.com/nf-core/tools/pull/1598))
+- Update how we interface with git remotes. ([#1626](https://github.com/nf-core/tools/issues/1626))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 
@@ -59,7 +60,7 @@
 - Add a new command `nf-core modules test` which runs pytests locally.
 - Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520))
 - Allow follow links when generating `test.yml` file with `nf-core modules create-test-yml` ([1570](https://github.com/nf-core/tools/pull/1570))
-- Escaped test run output before logging it, to avoid a rich ` MarkupError`
+- Escaped test run output before logging it, to avoid a rich `MarkupError`
 
 ### Linting
 

README.md

@@ -907,42 +907,25 @@ This allows multiple pipelines to use the same code for share tools and gives a
 
 The nf-core DSL2 modules repository is at <https://github.com/nf-core/modules>
 
+### Running the module commands
+
+The `nf-core/tools` package contains a suite of commands for working DSL2 modules in your pipeline. The modules commands use `git` to interface
+with the `nf-core/modules` repository. Therefore you need to have have `git` installed to use the modules commands. You can then use the
+the modules commands for a variety of tasks such as creating new DSL2 modules, viewing and getting metadata about the modules in your pipeline,
+and installing and updating modules from a remote git repository.
+
 ### Custom remote modules
 
 The modules supercommand comes with two flags for specifying a custom remote:
 
-- `--github-repository <github repo>`: Specify the repository from which the modules should be fetched. Defaults to `nf-core/modules`.
-- `--branch <branch name>`: Specify the branch from which the modules shoudl be fetched. Defaults to `master`.
+- `--git-remote <git remote url>`: Specify the repository from which the modules should be fetched as a git URL. Defaults to the github repository of `nf-core/modules`.
+- `--branch <branch name>`: Specify the branch from which the modules should be fetched. Defaults to the default branch of your repository
 
 Note that a custom remote must follow a similar directory structure to that of `nf-core/moduleś` for the `nf-core modules` commands to work properly.
 
-### Private remote modules
-
-In order to get access to your private modules repo, you need to create
-the `~/.config/gh/hosts.yml` file, which is the same file required by
-[GitHub CLI](https://cli.github.com/) to deal with private repositories.
-Such file is structured as follow:
-
-```conf
-github.com:
-    oauth_token: <your github access token>
-    user: <your github user>
-    git_protocol: <ssh or https are valid choices>
-```
-
-The easiest way to create this configuration file is through _GitHub CLI_: follow
-its [installation instructions](https://cli.github.com/manual/installation)
-and then call:
-
-```bash
-gh auth login
-```
-
-After that, you will be able to list and install your private modules without
-providing your github credentials through command line, by using `--github-repository`
-and `--branch` options properly.
-See the documentation on [gh auth login](https://cli.github.com/manual/gh_auth_login>)
-to get more information.
+The modules commands will during initalisation try to pull changes from the remote repositories. If you want to disable this, for example
+due to performance reason or if you want to run the commands offline, you can use the flag `--no-pull`. Note however that the commands will
+still need to clone repositories that have previously not been used.
 
 ### List modules
 
@@ -1079,7 +1062,7 @@ There are three additional flags that you can use when installing a module:
 
 - `--force`: Overwrite a previously installed version of the module.
 - `--prompt`: Select the module version using a cli prompt.
-- `--sha <commit_sha>`: Install the module at a specific commit from the `nf-core/modules` repository.
+- `--sha <commit_sha>`: Install the module at a specific commit.
 
 ### Update modules in a pipeline
 

nf_core/__main__.py

@@ -28,6 +28,9 @@
 # Submodules should all traverse back to this
 log = logging.getLogger()
 
+# Set up .nfcore directory for storing files between sessions
+nf_core.utils.setup_nfcore_dir()
+
 # Set up nicer formatting of click cli help messages
 click.rich_click.MAX_WIDTH = 100
 click.rich_click.USE_RICH_MARKUP = True
@@ -342,27 +345,31 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdow
 @nf_core_cli.group()
 @click.option(
     "-g",
-    "--github-repository",
+    "--git-remote",
     type=str,
-    default="nf-core/modules",
-    help="GitHub repository hosting modules.",
+    default=nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE,
+    help="Remote git repo to fetch files from",
+)
+@click.option("-b", "--branch", type=str, default=None, help="Branch of git repository hosting modules.")
+@click.option(
+    "--no-pull",
+    is_flag=True,
+    default=False,
+    help="Use this option for faster commands if you know there have been no recent changes to the repository",
 )
-@click.option("-b", "--branch", type=str, default="master", help="Branch of GitHub repository hosting modules.")
 @click.pass_context
-def modules(ctx, github_repository, branch):
+def modules(ctx, git_remote, branch, no_pull):
     """
     Commands to manage Nextflow DSL2 modules (tool wrappers).
     """
     # ensure that ctx.obj exists and is a dict (in case `cli()` is called
     # by means other than the `if` block below)
     ctx.ensure_object(dict)
 
-    # Make repository object to pass to subcommands
-    try:
-        ctx.obj["modules_repo_obj"] = nf_core.modules.ModulesRepo(github_repository, branch)
-    except LookupError as e:
-        log.critical(e)
-        sys.exit(1)
+    # Place the arguments in a context object
+    ctx.obj["modules_repo_url"] = git_remote
+    ctx.obj["modules_repo_branch"] = branch
+    ctx.obj["modules_repo_no_pull"] = no_pull
 
 
 # nf-core modules list subcommands
@@ -385,12 +392,17 @@ def remote(ctx, keywords, json):
     List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
     try:
-        module_list = nf_core.modules.ModuleList(None, remote=True)
+        module_list = nf_core.modules.ModuleList(
+            None, True, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         module_list.modules_repo = ctx.obj["modules_repo_obj"]
         print(module_list.list_modules(keywords, json))
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules list local
@@ -410,12 +422,16 @@ def local(ctx, keywords, json, dir):
     List modules installed locally in a pipeline
     """
     try:
-        module_list = nf_core.modules.ModuleList(dir, remote=False)
-        module_list.modules_repo = ctx.obj["modules_repo_obj"]
+        module_list = nf_core.modules.ModuleList(
+            dir, False, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         print(module_list.list_modules(keywords, json))
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules install
@@ -439,14 +455,24 @@ def install(ctx, tool, dir, prompt, force, sha):
     Fetches and installs module files from a remote repo e.g. nf-core/modules.
     """
     try:
-        module_install = nf_core.modules.ModuleInstall(dir, force=force, prompt=prompt, sha=sha)
-        module_install.modules_repo = ctx.obj["modules_repo_obj"]
+        module_install = nf_core.modules.ModuleInstall(
+            dir,
+            force,
+            prompt,
+            sha,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+        )
         exit_status = module_install.install(tool)
         if not exit_status and all:
             sys.exit(1)
     except UserWarning as e:
         log.error(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules update
@@ -487,15 +513,26 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff):
     """
     try:
         module_install = nf_core.modules.ModuleUpdate(
-            dir, force=force, prompt=prompt, sha=sha, update_all=all, show_diff=preview, save_diff_fn=save_diff
+            dir,
+            force,
+            prompt,
+            sha,
+            all,
+            preview,
+            save_diff,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
         )
-        module_install.modules_repo = ctx.obj["modules_repo_obj"]
         exit_status = module_install.update(tool)
         if not exit_status and all:
             sys.exit(1)
     except UserWarning as e:
         log.error(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules remove
@@ -514,12 +551,16 @@ def remove(ctx, dir, tool):
     Remove a module from a pipeline.
     """
     try:
-        module_remove = nf_core.modules.ModuleRemove(dir)
-        module_remove.modules_repo = ctx.obj["modules_repo_obj"]
+        module_remove = nf_core.modules.ModuleRemove(
+            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         module_remove.remove(tool)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules create
@@ -562,6 +603,9 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules create-test-yml
@@ -585,6 +629,9 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules lint
@@ -608,8 +655,9 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     nf-core/modules repository.
     """
     try:
-        module_lint = nf_core.modules.ModuleLint(dir=dir)
-        module_lint.modules_repo = ctx.obj["modules_repo_obj"]
+        module_lint = nf_core.modules.ModuleLint(
+            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         module_lint.lint(
             module=tool,
             key=key,
@@ -627,6 +675,9 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules info
@@ -653,12 +704,16 @@ def info(ctx, tool, dir):
     If not, usage from the remote modules repo will be shown.
     """
     try:
-        module_info = nf_core.modules.ModuleInfo(dir, tool)
-        module_info.modules_repo = ctx.obj["modules_repo_obj"]
+        module_info = nf_core.modules.ModuleInfo(
+            dir, tool, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         print(module_info.get_module_info())
     except UserWarning as e:
         log.error(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules bump-versions
@@ -674,14 +729,19 @@ def bump_versions(ctx, tool, dir, all, show_all):
     the nf-core/modules repo.
     """
     try:
-        version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=dir)
+        version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(
+            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all)
     except nf_core.modules.module_utils.ModuleException as e:
         log.error(e)
         sys.exit(1)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules mulled
@@ -745,6 +805,9 @@ def test_module(ctx, tool, no_prompts, pytest_args):
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core schema subcommands

nf_core/lint/modules_json.py

@@ -27,7 +27,14 @@ def modules_json(self):
         all_modules_passed = True
 
         for repo in modules_json["repos"].keys():
-            for key in modules_json["repos"][repo].keys():
+            # Check if the modules.json has been updated to keep the
+            if "modules" not in modules_json["repos"][repo] or "git_url" not in modules_json["repos"][repo]:
+                failed.append(
+                    f"Your `modules.json` file is outdated. Please remove it and reinstall it by running any module command"
+                )
+                continue
+
+            for key in modules_json["repos"][repo]["modules"]:
                 if not key in modules_command.module_names[repo]:
                     failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
                     all_modules_passed = False

nf_core/modules/bump_versions.py

@@ -28,8 +28,8 @@
 
 
 class ModuleVersionBumper(ModuleCommand):
-    def __init__(self, pipeline_dir):
-        super().__init__(pipeline_dir)
+    def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False):
+        super().__init__(pipeline_dir, remote_url, branch, no_pull)
 
         self.up_to_date = None
         self.updated = None