Merge pull request #2698 from nf-core/dev

dev -> master for 2.12

.editorconfig

@@ -10,3 +10,13 @@ indent_style = space
 
 [*.{md,yml,yaml,html,css,scss,js,cff}]
 indent_size = 2
+
+# ignore python and markdown files
+[*.py]
+indent_style = unset
+
+[**/{CONTRIBUTING,README}.md]
+indent_style = unset
+
+[**/Makefile]
+indent_style = unset

.github/.coveragerc

@@ -1,2 +1,5 @@
 [run]
-omit = nf_core/pipeline-template/*
+omit = nf_core/*-template/*
+source = nf_core
+relative_files = True
+

.github/CONTRIBUTING.md

@@ -35,45 +35,28 @@ pip install -e .
 
 ## Code formatting
 
-### Black
+### Ruff
 
-All Python code in nf-core/tools must be passed through the [Black Python code formatter](https://black.readthedocs.io/en/stable/).
+All Python code in nf-core/tools must be passed through the [Ruff code linter and formatter](https://github.com/astral-sh/ruff).
 This ensures a harmonised code formatting style throughout the package, from all contributors.
 
-You can run Black on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository:
+You can run Ruff on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository:
 
 ```bash
-black .
+ruff format .
 ```
 
-Alternatively, Black has [integrations for most common editors](https://black.readthedocs.io/en/stable/editor_integration.html)
+Alternatively, Ruff has [integrations for most common editors](https://github.com/astral-sh/ruff-lsp) and VSCode(https://github.com/astral-sh/ruff-vscode)
 to automatically format code when you hit save.
-You can also set it up to run when you [make a commit](https://black.readthedocs.io/en/stable/version_control_integration.html).
 
 There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
-any code does not adhere to Black formatting.
+any code does not adhere to Ruff formatting.
 
-### isort
-
-All Python code must also be passed through [isort](https://pycqa.github.io/isort/index.html).
-This ensures a harmonised imports throughout the package, from all contributors.
-
-To run isort on the command line recursively on the whole repository you can use:
-
-```bash
-isort .
-```
-
-isort also has [plugins for most common editors](https://github.com/pycqa/isort/wiki/isort-Plugins)
-to automatically format code when you hit save.
-Or [version control integration](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) to set it up to run when you make a commit.
-
-There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
-any code does not adhere to isort formatting.
+Ruff has been adopted for linting and formatting in replacement of Black, isort (for imports) and pyupgrade. It also includes Flake8.
 
 ### pre-commit hooks
 
-This repository comes with [pre-commit](https://pre-commit.com/) hooks for black, isort and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
+This repository comes with [pre-commit](https://pre-commit.com/) hooks for ruff and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
 
 You can use the pre-commit hooks if you like, but you don't have to. The CI on Github will run the same checks as the tools installed with pre-commit. If the pre-commit checks pass, then the same checks in the CI will pass, too.
 

.github/renovate.json5

@@ -1,5 +1,17 @@
 {
  $schema: "https://docs.renovatebot.com/renovate-schema.json",
  extends: ["github>nf-core/ops//.github/renovate/default.json5"],
+ ignorePaths: ["**/nf_core/pipeline-template/modules/nf-core/**"],
  baseBranches: ["dev"],
+ packageRules: [
+ {
+ matchDatasources: ["docker"],
+ matchPackageNames: ["python"],
+ versioning: "pep440",
+ },
+ {
+ matchDatasources: ["docker"],
+ registryUrls: ["docker.io"],
+ },
+ ],
 }

.github/workflows/branch.yml

@@ -18,7 +18,7 @@ jobs:
  # If the above check failed, post a comment on the PR explaining the failure
  - name: Post PR comment
  if: failure()
- uses: mshick/add-pr-comment@v1
+ uses: mshick/add-pr-comment@v2
  with:
  message: |
  ## This PR is against the `master` branch :x:

.github/workflows/changelog.py

@@ -0,0 +1,228 @@
+"""
+Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
+
+To be called by a CI action. Assumes the following environment variables are set:
+PR_TITLE, PR_NUMBER, GITHUB_WORKSPACE.
+
+Adds a line into the CHANGELOG.md:
+* Looks for the section to add the line to, based on the PR title, e.g. `Template:`, `Modules:`.
+* All other change will go under the "### General" section.
+* If an entry for the PR is already added, it will not run.
+
+Other assumptions:
+- CHANGELOG.md has a running section for an ongoing "dev" version
+(i.e. titled "## nf-core vX.Ydev").
+"""
+
+import os
+import re
+import sys
+from pathlib import Path
+from typing import List
+
+REPO_URL = "https://github.com/nf-core/tools"
+
+# Assumes the environment is set by the GitHub action.
+pr_title = os.environ["PR_TITLE"]
+pr_number = os.environ["PR_NUMBER"]
+comment = os.environ.get("COMMENT", "")
+workspace_path = Path(os.environ.get("GITHUB_WORKSPACE", ""))
+
+assert pr_title, pr_title
+assert pr_number, pr_number
+
+# Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
+pr_title = pr_title.removesuffix(f" (#{pr_number})")
+
+changelog_path = workspace_path / "CHANGELOG.md"
+
+if any(
+ line in pr_title.lower()
+ for line in [
+ "skip changelog",
+ "skip change log",
+ "no changelog",
+ "no change log",
+ "bump version",
+ ]
+):
+ print("Skipping changelog update")
+ sys.exit(0)
+
+
+def _determine_change_type(pr_title) -> tuple[str, str]:
+ """
+ Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
+ Returns a tuple of the section name and the module info.
+ """
+ sections = {
+ "Template": "### Template",
+ "Download": "### Download",
+ "Linting": "### Linting",
+ "Modules": "### Modules",
+ "Subworkflows": "### Subworkflows",
+ }
+ current_section_header = "### General"
+ current_section = "General"
+
+ # Check if the PR in any of the sections.
+ for section, section_header in sections.items():
+ # check if the PR title contains any of the section headers, with some loose matching, e.g. removing plural and suffixes
+ if re.sub(r"s$", "", section.lower().replace("ing", "")) in pr_title.lower():
+ current_section_header = section_header
+ current_section = section
+ print(f"Detected section: {current_section}")
+ return current_section, current_section_header
+
+
+# Determine the type of the PR
+section, section_header = _determine_change_type(pr_title)
+
+# Remove section indicator from the PR title.
+pr_title = re.sub(rf"{section}[:\s]*", "", pr_title, flags=re.IGNORECASE)
+
+# Prepare the change log entry.
+pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
+
+# Handle manual changelog entries through comments.
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+ print(f"Adding manual changelog entry: {comment}")
+ pr_title = comment
+new_lines = [
+ f"- {pr_title} {pr_link}\n",
+]
+
+print(f"Adding new lines into section '{section}':\n" + "".join(new_lines))
+
+# Finally, updating the changelog.
+# Read the current changelog lines. We will print them back as is, except for one new
+# entry, corresponding to this new PR.
+with changelog_path.open("r") as f:
+ orig_lines = f.readlines()
+updated_lines: List[str] = []
+
+
+def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> str:
+ if line.strip().endswith(pr_link):
+ print(f"Found existing entry for this pull request #{pr_number}:")
+ existing_lines = [line]
+ if new_lines and new_lines == existing_lines and same_section:
+ print(f"Found existing identical entry for this pull request #{pr_number} in the same section:")
+ print("".join(existing_lines))
+ sys.exit(0) # Just leaving the CHANGELOG intact
+ else:
+ print(
+ f"Found existing entry for this pull request #{pr_number}. It will be replaced and/or moved to proper section"
+ )
+ print("".join(existing_lines))
+ for _ in range(len(existing_lines)):
+ try:
+ line = orig_lines.pop(0)
+ except IndexError:
+ break
+ return line
+
+
+# Find the next line in the change log that matches the pattern "# nf-core/tools v.*dev"
+# If it doesn't exist, exist with code 1 (let's assume that a new section is added
+# manually or by CI when a release is pushed).
+# Else, find the next line that matches the `section` variable, and insert a new line
+# under it (we also assume that section headers are added already).
+inside_version_dev = False
+already_added_entry = False
+while orig_lines:
+ line = orig_lines.pop(0)
+
+ # If the line already contains a link to the PR, don't add it again.
+ line = _skip_existing_entry_for_this_pr(line, same_section=False)
+
+ if line.startswith("# ") and not line.strip() == "# nf-core/tools: Changelog": # Version header, e.g. "# v2.12dev"
+ print(f"Found version header: {line.strip()}")
+ updated_lines.append(line)
+
+ # Parse version from the line `# v2.12dev` or
+ # `# [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]` ...
+ if not (m := re.match(r".*(v\d+\.\d+(dev)?).*", line)):
+ print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr)
+ sys.exit(1)
+ version = m.group(1)
+
+ if not inside_version_dev:
+ if not version.endswith("dev"):
+ print(
+ "Can't find a 'dev' version section in the changelog. Make sure "
+ "it's created, and all the required sections, e.g. `### Template` are created under it .",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ inside_version_dev = True
+ else:
+ if version.endswith("dev"):
+ print(
+ f"Found another 'dev' version section in the changelog, make"
+ f"sure to change it to a 'release' stable version tag. "
+ f"Line: {line.strip()}",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ # We are past the dev version, so just add back the rest of the lines and break.
+ while orig_lines:
+ line = orig_lines.pop(0)
+ line = _skip_existing_entry_for_this_pr(line, same_section=False)
+ if line:
+ updated_lines.append(line)
+ break
+ continue
+ print(f"Found line: {line.strip()}")
+ print(f"inside_version_dev: {inside_version_dev}")
+ print(f"section_header: {section_header}")
+ if inside_version_dev and line.lower().startswith(section_header.lower()): # Section of interest header
+ print(f"Found section header: {line.strip()}")
+ if already_added_entry:
+ print(
+ f"Already added new lines into section {section}, is the section duplicated?",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ updated_lines.append(line)
+ # Collecting lines until the next section.
+ section_lines: List[str] = []
+ while True:
+ line = orig_lines.pop(0)
+ if line.startswith("#"):
+ print(f"Found the next section header: {line.strip()}")
+ # Found the next section header, so need to put all the lines we collected.
+ updated_lines.append("\n")
+ _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()]
+ updated_lines.extend(_updated_lines)
+ updated_lines.append("\n")
+ if new_lines:
+ print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines))
+ else:
+ print(f"Removed existing entry from {changelog_path} section '{section}'")
+ already_added_entry = True
+ # Pushing back the next section header line
+ orig_lines.insert(0, line)
+ break
+ # If the line already contains a link to the PR, don't add it again.
+ line = _skip_existing_entry_for_this_pr(line, same_section=True)
+ section_lines.append(line)
+ else:
+ updated_lines.append(line)
+
+
+def collapse_newlines(lines: List[str]) -> List[str]:
+ updated = []
+ for idx in range(len(lines)):
+ if idx != 0 and not lines[idx].strip() and not lines[idx - 1].strip():
+ continue
+ updated.append(lines[idx])
+ return updated
+
+
+updated_lines = collapse_newlines(updated_lines)
+
+
+# Finally, writing the updated lines back.
+with changelog_path.open("w") as f:
+ f.writelines(updated_lines)