From 0358f266eb81e759e795797cc0de59e845bfe653 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 26 May 2023 15:34:27 +0100 Subject: [PATCH 1/3] Adds public_aws_ecr profile for using ECR hosted images. Additions: - Adds public_aws_ecr profile which: - Sets the default registry to public.ecr.aws - Replaces all processes that aren't available on ECR with full URI --- .github/workflows/awsfulltest.yml | 2 +- .github/workflows/awstest.yml | 2 +- CHANGELOG.md | 1 + conf/public_aws_ecr.config | 60 +++++++++++++++++++++++++++++++ nextflow.config | 3 ++ 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 conf/public_aws_ecr.config diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index d731915357..84b2b282e4 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -25,7 +25,7 @@ jobs: { "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/somatic_test" } - profiles: test_full,aws_tower + profiles: test_full,public_aws_tower - uses: actions/upload-artifact@v3 with: name: Tower debug log file diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index ebcb22e0d3..6d7751e3d1 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -22,7 +22,7 @@ jobs: { "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-test-${{ github.sha }}" } - profiles: test,aws_tower + profiles: test,public_aws_tower - uses: actions/upload-artifact@v3 with: name: Tower debug log file diff --git a/CHANGELOG.md b/CHANGELOG.md index 155df2ac5f..6c265473d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#978](https://github.com/nf-core/sarek/pull/978) - Validate that patient/sample does not contain spaces - [#981](https://github.com/nf-core/sarek/pull/981) - Added documentation on generating ASCAT resources for exome and targeted sequencing - [#1041](https://github.com/nf-core/sarek/pull/1041) - Add params `vep_custom_args` to let user specify custom params more easily for `VEP` +- [null]() - Add `public_aws_ecr` for using ECR hosted containers. ### Changed diff --git a/conf/public_aws_ecr.config b/conf/public_aws_ecr.config new file mode 100644 index 0000000000..73f0a566f6 --- /dev/null +++ b/conf/public_aws_ecr.config @@ -0,0 +1,60 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + AWS ECR Config +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Config to set public AWS ECR images wherever possible + This improves speed when running on AWS infrastructure. + Use this as an example template when using your own private registry. +---------------------------------------------------------------------------------------- +*/ + +docker.registry = 'public.ecr.aws' +podman.registry = 'public.ecr.aws' + +process { + withName: 'ADD_INFO_TO_VCF' { + container = 'quay.io/biocontainers/gawk:5.1.0' + } + withName: 'ASCAT' { + container = 'quay.io/biocontainers/mulled-v2-c278c7398beb73294d78639a864352abef2931ce:ba3e6d2157eac2d38d22e62ec87675e12adb1010-0' + } + withName: 'BUILD_INTERVALS' { + container = 'quay.io/biocontainers/gawk:5.1.0' + } + withName: 'BWAMEM2_MEM' { + container = 'quay.io/biocontainers/mulled-v2-e5d375990341c5aef3c9aff74f96f66f65375ef6:2cdf6bf1e92acbeb9b2834b1c58754167173a410-0' + } + withName: 'BWA_MEM' { + container = 'quay.io/biocontainers/mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:219b6c272b25e7e642ae3ff0bf0c5c81a5135ab4-0' + } + withName: 'CAT_CAT' { + container = 'quay.io/biocontainers/pigz:2.3.4' + } + withName: 'CAT_FASTQ' { + container = 'quay.io/nf-core/ubuntu:20.04' + } + withName: 'CNVKIT_BATCH' { + container = 'quay.io/biocontainers/mulled-v2-780d630a9bb6a0ff2e7b6f730906fd703e40e98f:3bdd798e4b9aed6d3e1aaa1596c913a3eeb865cb-0' + } + withName: 'CREATE_INTERVALS_BED' { + container = 'quay.io/biocontainers/gawk:5.1.0' + } + withName: 'DRAGMAP_ALIGN' { + container = 'quay.io/biocontainers/mulled-v2-580d344d9d4a496cd403932da8765f9e0187774d:5ebebbc128cd624282eaa37d2c7fe01505a91a69-0' + } + withName: 'SAMBLASTER' { + container = 'quay.io/biocontainers/mulled-v2-19fa9f1a5c3966b63a24166365e81da35738c5ab:cee56b506ceb753d4bbef7e05b81e1bfc25d937f-0' + } + withName: 'SVDB_MERGE' { + container = 'quay.io/biocontainers/mulled-v2-c8daa8f9d69d3c5a1a4ff08283a166c18edb0000:af6f8534cd538a85ff43a2eae1b52b143e7abd05-0' + } + withName: 'UNTAR' { + container = 'quay.io/nf-core/ubuntu:20.04' + } + withName: 'UNZIP' { + container = 'quay.io/biocontainers/p7zip:16.02' + } + withName: 'VCFTOOLS' { + container = public.ecr.aws/biocontainers/vcftools:0.1.16--pl5321hd03093a_7 + } +} diff --git a/nextflow.config b/nextflow.config index c3aa5699b1..f39d3607a3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -235,6 +235,9 @@ profiles { executor.memory = 60.GB executor.name = 'local' } + public_aws_ecr { + includeConfig 'conf/public_aws_ecr.config' + } // Basic test profile for CI test { includeConfig 'conf/test/test.config' } // Extra test profiles for full tests on AWS From f58b9b70025ac0c96162bc1de3d65329b85ec07a Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 26 May 2023 15:42:49 +0100 Subject: [PATCH 2/3] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c265473d8..a59a636aef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#978](https://github.com/nf-core/sarek/pull/978) - Validate that patient/sample does not contain spaces - [#981](https://github.com/nf-core/sarek/pull/981) - Added documentation on generating ASCAT resources for exome and targeted sequencing - [#1041](https://github.com/nf-core/sarek/pull/1041) - Add params `vep_custom_args` to let user specify custom params more easily for `VEP` -- [null]() - Add `public_aws_ecr` for using ECR hosted containers. +- [#1045](https://github.com/nf-core/sarek/pull/1045) - Add `public_aws_ecr` for using ECR hosted containers. ### Changed From 08a38c964a12ae98dd90015555f1f37275b38d1f Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 26 May 2023 15:44:08 +0100 Subject: [PATCH 3/3] fixup --- conf/public_aws_ecr.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/public_aws_ecr.config b/conf/public_aws_ecr.config index 73f0a566f6..fb347256f9 100644 --- a/conf/public_aws_ecr.config +++ b/conf/public_aws_ecr.config @@ -55,6 +55,6 @@ process { container = 'quay.io/biocontainers/p7zip:16.02' } withName: 'VCFTOOLS' { - container = public.ecr.aws/biocontainers/vcftools:0.1.16--pl5321hd03093a_7 + container = 'public.ecr.aws/biocontainers/vcftools:0.1.16--pl5321hd03093a_7' } }