diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 3805c1a240..fb00471b90 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -38,6 +38,7 @@ jobs: - TEMPLATE: "template_skip_nf_core_configs.yml" runner: ubuntu-latest profile: "docker" + fail-fast: false steps: - name: go to working directory diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 9f117124a0..c58206158f 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -4,7 +4,7 @@ on: types: [created] jobs: - deploy: + fix-linting: # Only run if comment is on a PR with the main repo, and if it contains the magic keywords if: > contains(github.event.comment.html_url, '/pull/') && @@ -13,7 +13,7 @@ jobs: runs-on: self-hosted steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -31,11 +31,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - name: Set up Python 3.11 - uses: actions/setup-python@v5 + # Install and run pre-commit + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 - cache: "pip" - name: Install pre-commit run: pip install pre-commit @@ -47,15 +46,15 @@ jobs: # indication that the linting has finished - name: react if linting finished succesfully - if: ${{ steps.pre-commit.outcome }} == 'success' + if: steps.pre-commit.outcome == 'success' uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 with: comment-id: ${{ github.event.comment.id }} - reactions: green_check_mark + reactions: "+1" - name: Commit & push changes id: commit-and-push - if: ${{ steps.pre-commit.outcome }} == 'failure' + if: steps.pre-commit.outcome == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" @@ -66,17 +65,25 @@ jobs: git push - name: react if linting errors were fixed - if: ${{ steps.commit-and-push.outcome }} == 'success' + id: react-if-fixed + if: steps.commit-and-push.outcome == 'success' uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 with: comment-id: ${{ github.event.comment.id }} - reactions: pencil2 + reactions: hooray - name: react if linting errors were not fixed - if: ${{ steps.commit-and-push.outcome }} == 'failure' + if: steps.commit-and-push.outcome == 'failure' uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 with: comment-id: ${{ github.event.comment.id }} - reactions: x + reactions: confused + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3 + with: + issue-number: ${{ github.event.issue.number }} body: | @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. + See [CI log](https://github.com/nf-core/tools/actions/runs/${{ github.run_id }}) for more details. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56ebb5bfaa..d4eb6a721d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.13 + rev: v0.1.14 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix diff --git a/CHANGELOG.md b/CHANGELOG.md index 2222491165..09bb35c869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,10 @@ - switch to new image syntax in readme ([#2645](https://github.com/nf-core/tools/pull/2645)) - Add conda channel order to nextflow.config ([#2094](https://github.com/nf-core/tools/pull/2094)) - Fix tyop in pipeline nextflow.config ([#2664](https://github.com/nf-core/tools/pull/2664)) -- add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678)) +- Remove `nfcore_external_java_deps.jar` from lib directory in pipeline template ([#2675](https://github.com/nf-core/tools/pull/2675)) +- Add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678)) +- Add new pipeline error message pointing to docs when 'requirement exceeds available memory' error message ([#2680](https://github.com/nf-core/tools/pull/2680)) +- add 👀👍🏻🎉😕 reactions to fix-linting-bot action ([#2692](https://github.com/nf-core/tools/pull/2692)) ### Download @@ -19,6 +22,7 @@ - Fix linting of a pipeline with patched custom module ([#2669](https://github.com/nf-core/tools/pull/2669)) - linting a pipeline also lints the installed subworkflows ([#2677](https://github.com/nf-core/tools/pull/2677)) - environment.yml name must be lowercase ([#2676](https://github.com/nf-core/tools/pull/2676)) +- lint `nextflow.config` default values match the ones specified in `nextflow_schema.json` ([#2684](https://github.com/nf-core/tools/pull/2684)) ### Modules @@ -39,7 +43,10 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.1.13 ([#2660](https://github.com/nf-core/tools/pull/2660)) - Add new subcommand: `nf-core logo-create` to output an nf-core logo for a pipeline (instead of going through the website) ([#2662](https://github.com/nf-core/tools/pull/2662)) - Update actions/cache action to v4 ([#2666](https://github.com/nf-core/tools/pull/2666)) +- Handle api redirects from the old site ([#2672](https://github.com/nf-core/tools/pull/2672)) - Remove redundanct v in pipeline version for emails ([#2667](https://github.com/nf-core/tools/pull/2667)) +- add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.1.14 ([#2674](https://github.com/nf-core/tools/pull/2674)) - Update peter-evans/create-or-update-comment action to v4 ([#2683](https://github.com/nf-core/tools/pull/2683)) # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20] diff --git a/nf_core/create.py b/nf_core/create.py index 76fc50e8fa..f9e9933ee3 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -505,10 +505,12 @@ def fix_linting(self): def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" email_logo_path = Path(self.outdir) / "assets" - create_logo(text=self.template_params["short_name"], dir=email_logo_path, theme="light") + create_logo(text=self.template_params["short_name"], dir=email_logo_path, theme="light", force=self.force) for theme in ["dark", "light"]: readme_logo_path = Path(self.outdir) / "docs" / "images" - create_logo(text=self.template_params["short_name"], dir=readme_logo_path, width=600, theme=theme) + create_logo( + text=self.template_params["short_name"], dir=readme_logo_path, width=600, theme=theme, force=self.force + ) def git_init_pipeline(self): """Initialises the new pipeline as a Git repository and submits first commit. @@ -537,8 +539,24 @@ def git_init_pipeline(self): repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") if default_branch: repo.active_branch.rename(default_branch) - repo.git.branch("TEMPLATE") - repo.git.branch("dev") + try: + repo.git.branch("TEMPLATE") + repo.git.branch("dev") + + except git.GitCommandError as e: + if "already exists" in e.stderr: + log.debug("Branches 'TEMPLATE' and 'dev' already exist") + if self.force: + log.debug("Force option set - deleting branches") + repo.git.branch("-D", "TEMPLATE") + repo.git.branch("-D", "dev") + repo.git.branch("TEMPLATE") + repo.git.branch("dev") + else: + log.error( + "Branches 'TEMPLATE' and 'dev' already exist. Use --force to overwrite existing branches." + ) + sys.exit(1) log.info( "Done. Remember to add a remote and push to GitHub:\n" f"[white on grey23] cd {self.outdir} \n" diff --git a/nf_core/create_logo.py b/nf_core/create_logo.py index e40dd73452..4dfebd3712 100644 --- a/nf_core/create_logo.py +++ b/nf_core/create_logo.py @@ -5,6 +5,7 @@ from PIL import Image, ImageDraw, ImageFont import nf_core +from nf_core.utils import NFCORE_CACHE_DIR log = logging.getLogger(__name__) @@ -26,7 +27,6 @@ def create_logo( if not dir.is_dir(): log.debug(f"Creating directory {dir}") dir.mkdir(parents=True, exist_ok=True) - assets = Path(nf_core.__file__).parent / "assets/logo" if format == "svg": @@ -51,44 +51,56 @@ def create_logo( else: logo_filename = f"nf-core-{text}_logo_{theme}.png" if not filename else filename logo_filename = f"{logo_filename}.png" if not logo_filename.lower().endswith(".png") else logo_filename + cache_name = f"nf-core-{text}_logo_{theme}_{width}.png" logo_path = Path(dir, logo_filename) # Check if we haven't already created this logo if logo_path.is_file() and not force: log.info(f"Logo already exists at: {logo_path}. Use `--force` to overwrite.") return logo_path - - log.debug(f"Creating logo for {text}") - - # make sure the figure fits the text - font_path = assets / "MavenPro-Bold.ttf" - log.debug(f"Using font: {str(font_path)}") - font = ImageFont.truetype(str(font_path), 400) - text_length = font.getmask(text).getbbox()[2] # get the width of the text based on the font - - max_width = max( - 2300, text_length + len(text) * 20 - ) # need to add some more space to the text length to make sure it fits - - template_fn = "nf-core-repo-logo-base-lightbg.png" - if theme == "dark": - template_fn = "nf-core-repo-logo-base-darkbg.png" - - template_path = assets / template_fn - img = Image.open(str(template_path)) - # get the height of the template image - height = img.size[1] - - # Draw text - draw = ImageDraw.Draw(img) - color = theme == "dark" and (250, 250, 250) or (5, 5, 5) - draw.text((110, 465), text, color, font=font) - - # Crop to max width - img = img.crop((0, 0, max_width, height)) - - # Resize - img = img.resize((width, int((width / max_width) * height))) + # cache file + cache_path = Path(NFCORE_CACHE_DIR, "logo", cache_name) + img = None + if cache_path.is_file(): + log.debug(f"Logo already exists in cache at: {cache_path}. Reusing this file.") + img = Image.open(str(cache_path)) + if not img: + log.debug(f"Creating logo for {text}") + + # make sure the figure fits the text + font_path = assets / "MavenPro-Bold.ttf" + log.debug(f"Using font: {str(font_path)}") + font = ImageFont.truetype(str(font_path), 400) + text_length = font.getmask(text).getbbox()[2] # get the width of the text based on the font + + max_width = max( + 2300, text_length + len(text) * 20 + ) # need to add some more space to the text length to make sure it fits + + template_fn = "nf-core-repo-logo-base-lightbg.png" + if theme == "dark": + template_fn = "nf-core-repo-logo-base-darkbg.png" + + template_path = assets / template_fn + img = Image.open(str(template_path)) + # get the height of the template image + height = img.size[1] + + # Draw text + draw = ImageDraw.Draw(img) + color = theme == "dark" and (250, 250, 250) or (5, 5, 5) + draw.text((110, 465), text, color, font=font) + + # Crop to max width + img = img.crop((0, 0, max_width, height)) + + # Resize + img = img.resize((width, int((width / max_width) * height))) + + # Save to cache + Path(cache_path.parent).mkdir(parents=True, exist_ok=True) + log.debug(f"Saving logo to cache: {cache_path}") + img.save(cache_path, "PNG") # Save img.save(logo_path, "PNG") diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index b4d0c10ff3..4f7657aec5 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -120,8 +120,10 @@ def run_linting( if subworkflow_lint_obj is not None: subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) - # Set up files for modules linting test + # Set up files for component linting test module_lint_obj.set_up_pipeline_files() + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.set_up_pipeline_files() # Run the pipeline linting tests try: diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 117704d1f1..707f98a53a 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -1,5 +1,6 @@ import logging -import os +from pathlib import Path +from typing import Union log = logging.getLogger(__name__) @@ -51,7 +52,6 @@ def files_exist(self): docs/output.md docs/README.md docs/usage.md - lib/nfcore_external_java_deps.jar lib/NfcoreTemplate.groovy lib/Utils.groovy lib/WorkflowMain.groovy @@ -98,6 +98,12 @@ def files_exist(self): .travis.yml + Files that *must not* be present if a certain entry is present in ``nextflow.config``: + + .. code-block:: bash + + lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config + .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting the ``files_exist`` key as follows in your ``.nf-core.yml`` config file. For example: @@ -132,48 +138,46 @@ def files_exist(self): ["CHANGELOG.md"], ["CITATIONS.md"], ["CODE_OF_CONDUCT.md"], - ["CODE_OF_CONDUCT.md"], ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling ["nextflow_schema.json"], ["nextflow.config"], ["README.md"], - [os.path.join(".github", ".dockstore.yml")], - [os.path.join(".github", "CONTRIBUTING.md")], - [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")], - [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")], - [os.path.join(".github", "workflows", "branch.yml")], - [os.path.join(".github", "workflows", "ci.yml")], - [os.path.join(".github", "workflows", "linting_comment.yml")], - [os.path.join(".github", "workflows", "linting.yml")], - [os.path.join("assets", "email_template.html")], - [os.path.join("assets", "email_template.txt")], - [os.path.join("assets", "sendmail_template.txt")], - [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("conf", "modules.config")], - [os.path.join("conf", "test.config")], - [os.path.join("conf", "test_full.config")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")], - [os.path.join("docs", "output.md")], - [os.path.join("docs", "README.md")], - [os.path.join("docs", "README.md")], - [os.path.join("docs", "usage.md")], - [os.path.join("lib", "nfcore_external_java_deps.jar")], - [os.path.join("lib", "NfcoreTemplate.groovy")], - [os.path.join("lib", "Utils.groovy")], - [os.path.join("lib", "WorkflowMain.groovy")], + [Path(".github", ".dockstore.yml")], + [Path(".github", "CONTRIBUTING.md")], + [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")], + [Path(".github", "ISSUE_TEMPLATE", "config.yml")], + [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")], + [Path(".github", "PULL_REQUEST_TEMPLATE.md")], + [Path(".github", "workflows", "branch.yml")], + [Path(".github", "workflows", "ci.yml")], + [Path(".github", "workflows", "linting_comment.yml")], + [Path(".github", "workflows", "linting.yml")], + [Path("assets", "email_template.html")], + [Path("assets", "email_template.txt")], + [Path("assets", "sendmail_template.txt")], + [Path("assets", f"nf-core-{short_name}_logo_light.png")], + [Path("conf", "modules.config")], + [Path("conf", "test.config")], + [Path("conf", "test_full.config")], + [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")], + [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")], + [Path("docs", "output.md")], + [Path("docs", "README.md")], + [Path("docs", "README.md")], + [Path("docs", "usage.md")], + [Path("lib", "NfcoreTemplate.groovy")], + [Path("lib", "Utils.groovy")], + [Path("lib", "WorkflowMain.groovy")], ] files_warn = [ ["main.nf"], - [os.path.join("assets", "multiqc_config.yml")], - [os.path.join("conf", "base.config")], - [os.path.join("conf", "igenomes.config")], - [os.path.join(".github", "workflows", "awstest.yml")], - [os.path.join(".github", "workflows", "awsfulltest.yml")], - [os.path.join("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")], + [Path("assets", "multiqc_config.yml")], + [Path("conf", "base.config")], + [Path("conf", "igenomes.config")], + [Path(".github", "workflows", "awstest.yml")], + [Path(".github", "workflows", "awsfulltest.yml")], + [Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")], ["modules.json"], ["pyproject.toml"], ] @@ -184,45 +188,48 @@ def files_exist(self): "parameters.settings.json", "pipeline_template.yml", # saving information in .nf-core.yml ".nf-core.yaml", # yml not yaml - os.path.join("bin", "markdown_to_html.r"), - os.path.join("conf", "aws.config"), - os.path.join(".github", "workflows", "push_dockerhub.yml"), - os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.md"), - os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.md"), - os.path.join("docs", "images", f"nf-core-{short_name}_logo.png"), + Path("bin", "markdown_to_html.r"), + Path("conf", "aws.config"), + Path(".github", "workflows", "push_dockerhub.yml"), + Path(".github", "ISSUE_TEMPLATE", "bug_report.md"), + Path(".github", "ISSUE_TEMPLATE", "feature_request.md"), + Path("docs", "images", f"nf-core-{short_name}_logo.png"), ".markdownlint.yml", ".yamllint.yml", - os.path.join("lib", "Checks.groovy"), - os.path.join("lib", "Completion.groovy"), - os.path.join("lib", "Workflow.groovy"), + Path("lib", "Checks.groovy"), + Path("lib", "Completion.groovy"), + Path("lib", "Workflow.groovy"), ] files_warn_ifexists = [".travis.yml"] + files_fail_ifinconfig = [[Path("lib", "nfcore_external_java_deps.jar"), "nf-validation"]] # Remove files that should be ignored according to the linting config ignore_files = self.lint_config.get("files_exist", []) + log.info(f"Files to ignore: {ignore_files}") - def pf(file_path): - return os.path.join(self.wf_path, file_path) + def pf(file_path: Union[str, Path]) -> Path: + return Path(self.wf_path, file_path) # First - critical files. Check that this is actually a Nextflow pipeline - if not os.path.isfile(pf("nextflow.config")) and not os.path.isfile(pf("main.nf")): + if not pf("nextflow.config").is_file() and not pf("main.nf").is_file(): failed.append("File not found: nextflow.config or main.nf") raise AssertionError("Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?") # Files that cause an error if they don't exist for files in files_fail: - if any([f in ignore_files for f in files]): + print(files) + if any([str(f) in ignore_files for f in files]): continue - if any([os.path.isfile(pf(f)) for f in files]): + if any([pf(f).is_file() for f in files]): passed.append(f"File found: {self._wrap_quotes(files)}") else: failed.append(f"File not found: {self._wrap_quotes(files)}") # Files that cause a warning if they don't exist for files in files_warn: - if any([f in ignore_files for f in files]): + if any([str(f) in ignore_files for f in files]): continue - if any([os.path.isfile(pf(f)) for f in files]): + if any([pf(f).is_file() for f in files]): passed.append(f"File found: {self._wrap_quotes(files)}") else: warned.append(f"File not found: {self._wrap_quotes(files)}") @@ -231,16 +238,32 @@ def pf(file_path): for file in files_fail_ifexists: if file in ignore_files: continue - if os.path.isfile(pf(file)): + if pf(file).is_file(): failed.append(f"File must be removed: {self._wrap_quotes(file)}") else: passed.append(f"File not found check: {self._wrap_quotes(file)}") - + # Files that cause an error if they exists together with a certain entry in nextflow.config + for file in files_fail_ifinconfig: + if str(file[0]) in ignore_files: + continue + nextflow_config = pf("nextflow.config") + in_config = False + with open(nextflow_config) as f: + if file[1] in f.read(): + in_config = True + if pf(file[0]).is_file() and in_config: + failed.append(f"File must be removed: {self._wrap_quotes(file[0])}") + elif pf(file[0]).is_file() and not in_config: + passed.append(f"File found check: {self._wrap_quotes(file[0])}") + elif not pf(file[0]).is_file() and not in_config: + failed.append(f"File not found check: {self._wrap_quotes(file[0])}") + elif not pf(file[0]).is_file() and in_config: + passed.append(f"File not found check: {self._wrap_quotes(file[0])}") # Files that cause a warning if they exist for file in files_warn_ifexists: if file in ignore_files: continue - if os.path.isfile(pf(file)): + if pf(file).is_file(): warned.append(f"File should be removed: {self._wrap_quotes(file)}") else: passed.append(f"File not found check: {self._wrap_quotes(file)}") diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 82b286fb44..176b0e9e65 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -1,8 +1,9 @@ import filecmp import logging -import os import shutil import tempfile +from pathlib import Path +from typing import Union import yaml @@ -39,7 +40,6 @@ def files_unchanged(self): docs/images/nf-core-PIPELINE_logo_light.png docs/images/nf-core-PIPELINE_logo_dark.png docs/README.md' - lib/nfcore_external_java_deps.jar lib/NfcoreTemplate.groovy ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling @@ -49,6 +49,10 @@ def files_unchanged(self): .prettierignore pyproject.toml + Files that need to be there or not based on a entry in nextflow config:: + + lib/nfcore_external_java_deps.jar # if config doesn't mention nf-validation + .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting the ``files_unchanged`` key as follows in your ``.nf-core.yml`` config file. For example: @@ -87,28 +91,30 @@ def files_unchanged(self): [".prettierrc.yml"], ["CODE_OF_CONDUCT.md"], ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling - [os.path.join(".github", ".dockstore.yml")], - [os.path.join(".github", "CONTRIBUTING.md")], - [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")], - [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")], - [os.path.join(".github", "workflows", "branch.yml")], - [os.path.join(".github", "workflows", "linting_comment.yml")], - [os.path.join(".github", "workflows", "linting.yml")], - [os.path.join("assets", "email_template.html")], - [os.path.join("assets", "email_template.txt")], - [os.path.join("assets", "sendmail_template.txt")], - [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")], - [os.path.join("docs", "README.md")], - [os.path.join("lib", "nfcore_external_java_deps.jar")], - [os.path.join("lib", "NfcoreTemplate.groovy")], + [Path(".github", ".dockstore.yml")], + [Path(".github", "CONTRIBUTING.md")], + [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")], + [Path(".github", "ISSUE_TEMPLATE", "config.yml")], + [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")], + [Path(".github", "PULL_REQUEST_TEMPLATE.md")], + [Path(".github", "workflows", "branch.yml")], + [Path(".github", "workflows", "linting_comment.yml")], + [Path(".github", "workflows", "linting.yml")], + [Path("assets", "email_template.html")], + [Path("assets", "email_template.txt")], + [Path("assets", "sendmail_template.txt")], + [Path("assets", f"nf-core-{short_name}_logo_light.png")], + [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")], + [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")], + [Path("docs", "README.md")], + [Path("lib", "NfcoreTemplate.groovy")], ] files_partial = [ [".gitignore", ".prettierignore", "pyproject.toml"], ] + files_conditional = [ + [Path("lib", "nfcore_external_java_deps.jar"), {"plugins": "nf_validation"}], + ] # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) @@ -124,24 +130,24 @@ def files_unchanged(self): "prefix": prefix, } - template_yaml_path = os.path.join(tmp_dir, "template.yaml") + template_yaml_path = Path(tmp_dir, "template.yaml") with open(template_yaml_path, "w") as fh: yaml.dump(template_yaml, fh, default_flow_style=False) - test_pipeline_dir = os.path.join(tmp_dir, f"{prefix}-{short_name}") + test_pipeline_dir = Path(tmp_dir, f"{prefix}-{short_name}") create_obj = nf_core.create.PipelineCreate( None, None, None, no_git=True, outdir=test_pipeline_dir, template_yaml_path=template_yaml_path ) create_obj.init_pipeline() # Helper functions for file paths - def _pf(file_path): + def _pf(file_path: Union[str, Path]) -> Path: """Helper function - get file path for pipeline file""" - return os.path.join(self.wf_path, file_path) + return Path(self.wf_path, file_path) - def _tf(file_path): + def _tf(file_path: Union[str, Path]) -> Path: """Helper function - get file path for template file""" - return os.path.join(test_pipeline_dir, file_path) + return Path(test_pipeline_dir, file_path) # Files that must be completely unchanged from template for files in files_exact: @@ -151,7 +157,7 @@ def _tf(file_path): ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file - elif not any([os.path.isfile(_pf(f)) for f in files]): + elif not any([_pf(f).is_file() for f in files]): ignored.append(f"File does not exist: {self._wrap_quotes(files)}") # Check that the file has an identical match @@ -180,7 +186,7 @@ def _tf(file_path): ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file - elif not any([os.path.isfile(_pf(f)) for f in files]): + elif not any([_pf(f).is_file() for f in files]): ignored.append(f"File does not exist: {self._wrap_quotes(files)}") # Check that the file contains the template file contents @@ -208,6 +214,39 @@ def _tf(file_path): except FileNotFoundError: pass + # Files that should be there only if an entry in nextflow config is not set + for files in files_conditional: + # Ignore if file specified in linting config + ignore_files = self.lint_config.get("files_unchanged", []) + if files[0] in ignore_files: + ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") + + # Ignore if we can't find the file + elif _pf(files[0]).is_file(): + ignored.append(f"File does not exist: {self._wrap_quotes(files[0])}") + + # Check that the file has an identical match + else: + config_key, config_value = list(files[1].items())[0] + if config_key in self.nf_config and self.nf_config[config_key] == config_value: + # Ignore if the config key is set to the expected value + ignored.append(f"File ignored due to config: {self._wrap_quotes(files)}") + else: + try: + if filecmp.cmp(_pf(files[0]), _tf(files[0]), shallow=True): + passed.append(f"`{files[0]}` matches the template") + else: + if "files_unchanged" in self.fix: + # Try to fix the problem by overwriting the pipeline file + shutil.copy(_tf(files[0]), _pf(files[0])) + passed.append(f"`{files[0]}` matches the template") + fixed.append(f"`{files[0]}` overwritten with template file") + else: + failed.append(f"`{files[0]}` does not match the template") + could_fix = True + except FileNotFoundError: + pass + # cleaning up temporary dir shutil.rmtree(tmp_dir) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 328bc03759..1e0a6c4995 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -1,6 +1,9 @@ import logging import os import re +from pathlib import Path + +from nf_core.schema import PipelineSchema log = logging.getLogger(__name__) @@ -113,6 +116,18 @@ def nextflow_config(self): * A ``test`` configuration profile should exist. + **The default values in ``nextflow.config`` should match the default values defined in the ``nextflow_schema.json``.** + + .. tip:: You can choose to ignore tests for the default value of an specific parameter + by creating a file called ``.nf-core.yml`` in the root of your pipeline and creating + a list the config parameters that should be ignored. For example to ignore the default value for the input parameter: + + .. code-block:: yaml + + lint: + nextflow_config: + - config_defaults: + - params.input """ passed = [] warned = [] @@ -347,4 +362,40 @@ def nextflow_config(self): else: failed.append("nextflow.config does not contain configuration profile `test`") + # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json + ignore_defaults = [] + for item in ignore_configs: + if isinstance(item, dict) and "config_defaults" in item: + ignore_defaults = item.get("config_defaults", []) + schema_path = Path(self.wf_path) / "nextflow_schema.json" + schema = PipelineSchema() + schema.schema_filename = schema_path + schema.no_prompts = True + schema.load_schema() + schema.get_schema_defaults() # Get default values from schema + self.nf_config.keys() # Params in nextflow.config + for param_name in schema.schema_defaults.keys(): + param = "params." + param_name + # Convert booleans to strings if needed + schema_default = ( + "true" + if str(schema.schema_defaults[param_name]) == "True" + else "false" + if str(schema.schema_defaults[param_name]) == "False" + else str(schema.schema_defaults[param_name]) + ) + if param in ignore_defaults: + ignored.append(f"Config default ignored: {param}") + elif param in self.nf_config.keys(): + if str(self.nf_config[param]) == schema_default: + passed.append(f"Config default value correct: {param}") + else: + failed.append( + f"Config default value incorrect: `{param}` is set as {self._wrap_quotes(schema_default)} in `nextflow_schema.json` but is {self._wrap_quotes(self.nf_config[param])} in `nextflow.config`." + ) + else: + failed.append( + f"Default value from the Nextflow schema '{param} = {self._wrap_quotes(schema_default)}' not found in `nextflow.config`." + ) + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index d9986bd30f..71f235ef41 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -4,7 +4,7 @@ on: types: [created] jobs: - deploy: + fix-linting: # Only run if comment is on a PR with the main repo, and if it contains the magic keywords if: > contains(github.event.comment.html_url, '/pull/') && @@ -13,10 +13,17 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} + # indication that the linting is being fixed + - name: React on comment + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: eyes + # Action runs on the issue comment, so we don't get the PR by default # Use the gh cli to check out the PR - name: Checkout Pull Request @@ -24,25 +31,59 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - name: Set up Python 3.11 - uses: actions/setup-python@v5 + # Install and run pre-commit + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 - cache: "pip" - name: Install pre-commit run: pip install pre-commit - name: Run pre-commit - run: pre-commit run --all-files || echo "status=fail" >> $GITHUB_ENV + id: pre-commit + run: pre-commit run --all-files + continue-on-error: true + + # indication that the linting has finished + - name: react if linting finished succesfully + if: steps.pre-commit.outcome == 'success' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: "+1" - name: Commit & push changes - if: env.status == 'fail' + id: commit-and-push + if: steps.pre-commit.outcome == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git config push.default upstream git add . git status - git commit -m "[automated] Fix linting with pre-commit" - git push {%- endraw %} + git commit -m "[automated] Fix code linting" + git push + + - name: react if linting errors were fixed + id: react-if-fixed + if: steps.commit-and-push.outcome == 'success' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: hooray + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: confused + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3 + with: + issue-number: ${{ github.event.issue.number }} + body: | + @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. + See [CI log](https://github.com/{% endraw %}{{name}}{% raw %}/actions/runs/${{ github.run_id }}) for more details.{% endraw %} diff --git a/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar b/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar deleted file mode 100644 index 805c8bb5e4..0000000000 Binary files a/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar and /dev/null differ diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 984c091091..7d08e1c8ef 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -1,4 +1,4 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Ruff. +# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. [tool.ruff] line-length = 120 diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 558e9a1f9f..4583f2a9d6 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -127,6 +127,13 @@ workflow.onComplete { } } +workflow.onError { + if (workflow.errorReport.contains("Process requirement exceeds available memory")) { + println("🛑 Default resources exceed availability 🛑 ") + println("💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡") + } +} + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END diff --git a/nf_core/utils.py b/nf_core/utils.py index 0b54056cfd..0c0590b0a3 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -420,11 +420,14 @@ def poll_nfcore_web_api(api_url, post_data=None): except requests.exceptions.ConnectionError: raise AssertionError(f"Could not connect to URL: {api_url}") else: - if response.status_code != 200: + if response.status_code != 200 and response.status_code != 301: log.debug(f"Response content:\n{response.content}") raise AssertionError( f"Could not access remote API results: {api_url} (HTML {response.status_code} Error)" ) + # follow redirects + if response.status_code == 301: + return poll_nfcore_web_api(response.headers["Location"], post_data) try: web_response = json.loads(response.content) if "status" not in web_response: diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py index 4e5e4d3c2b..5ba26d77a0 100644 --- a/tests/lint/files_exist.py +++ b/tests/lint/files_exist.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import nf_core.lint @@ -7,7 +8,7 @@ def test_files_exist_missing_config(self): """Lint test: critical files missing FAIL""" new_pipeline = self._make_pipeline_copy() - os.remove(os.path.join(new_pipeline, "CHANGELOG.md")) + Path(new_pipeline, "CHANGELOG.md").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() @@ -21,7 +22,7 @@ def test_files_exist_missing_main(self): """Check if missing main issues warning""" new_pipeline = self._make_pipeline_copy() - os.remove(os.path.join(new_pipeline, "main.nf")) + Path(new_pipeline, "main.nf").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() @@ -34,7 +35,7 @@ def test_files_exist_depreciated_file(self): """Check whether depreciated file issues warning""" new_pipeline = self._make_pipeline_copy() - nf = os.path.join(new_pipeline, "parameters.settings.json") + nf = Path(new_pipeline, "parameters.settings.json") os.system(f"touch {nf}") lint_obj = nf_core.lint.PipelineLint(new_pipeline) diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index 5d5f8e7345..60aaee5243 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -1,5 +1,6 @@ import os import re +from pathlib import Path import nf_core.create import nf_core.lint @@ -53,3 +54,66 @@ def test_nextflow_config_missing_test_profile_failed(self): result = lint_obj.nextflow_config() assert len(result["failed"]) > 0 assert len(result["warned"]) == 0 + + +def test_default_values_match(self): + """Test that the default values in nextflow.config match the default values defined in the nextflow_schema.json.""" + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + assert len(result["warned"]) == 0 + assert "Config default value correct: params.max_cpus" in result["passed"] + assert "Config default value correct: params.validate_params" in result["passed"] + + +def test_default_values_fail(self): + """Test linting fails if the default values in nextflow.config do not match the ones defined in the nextflow_schema.json.""" + new_pipeline = self._make_pipeline_copy() + # Change the default value of max_cpus in nextflow.config + nf_conf_file = Path(new_pipeline) / "nextflow.config" + with open(nf_conf_file) as f: + content = f.read() + fail_content = re.sub(r"\bmax_cpus = 16\b", "max_cpus = 0", content) + with open(nf_conf_file, "w") as f: + f.write(fail_content) + # Change the default value of max_memory in nextflow_schema.json + nf_schema_file = Path(new_pipeline) / "nextflow_schema.json" + with open(nf_schema_file) as f: + content = f.read() + fail_content = re.sub(r'"default": "128.GB"', '"default": "18.GB"', content) + print(fail_content) + with open(nf_schema_file, "w") as f: + f.write(fail_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 2 + assert ( + "Config default value incorrect: `params.max_cpus` is set as `16` in `nextflow_schema.json` but is `0` in `nextflow.config`." + in result["failed"] + ) + assert ( + "Config default value incorrect: `params.max_memory` is set as `18.GB` in `nextflow_schema.json` but is `128.GB` in `nextflow.config`." + in result["failed"] + ) + + +def test_default_values_ignored(self): + """Test ignoring linting of default values.""" + new_pipeline = self._make_pipeline_copy() + # Add max_cpus to the ignore list + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml, "w") as f: + f.write( + "repository_type: pipeline\nlint:\n nextflow_config:\n - config_defaults:\n - params.max_cpus\n" + ) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + lint_obj._load_lint_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + assert len(result["ignored"]) == 1 + assert "Config default value correct: params.max_cpus" not in result["passed"] + assert "Config default ignored: params.max_cpus" in result["ignored"] diff --git a/tests/test_lint.py b/tests/test_lint.py index 32913bda0d..ff7e56744d 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -219,6 +219,9 @@ def test_sphinx_md_files(self): test_multiqc_incorrect_export_plots, ) from .lint.nextflow_config import ( # type: ignore[misc] + test_default_values_fail, + test_default_values_ignored, + test_default_values_match, test_nextflow_config_bad_name_fail, test_nextflow_config_dev_in_release_mode_failed, test_nextflow_config_example_pass,