From 6930408ba1dba6e70ddcf76fa713d2533a007af5 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 6 Mar 2022 17:47:40 +0100 Subject: [PATCH] Fix Reduce task command runner script size [ci fast] This commit mitigate the problem of having a too big command runner script, skipping the adding of input files deletion commands in the script itself, when the number of input files is greater than or equals 100. Solve #2118 --- .../executor/SimpleFileCopyStrategy.groovy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy index 5eacfd32b3..40f58eb714 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy @@ -114,14 +114,24 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy { String getStageInputFilesScript(Map inputFiles) { assert inputFiles != null + def len = inputFiles.size() def delete = [] def links = [] for( Map.Entry entry : inputFiles ) { final stageName = entry.key final storePath = entry.value - // delete all previous files with the same name - delete << "rm -f ${Escape.path(stageName)}" + // Delete all previous files with the same name + // Note: the file deletion is only needed to prevent + // file name collisions when re-running the runner script + // for debugging purpose. However, this can cause the creation + // of a very big runner script when a large number of files is + // given due to the file name duplication. Therefore the rationale + // here is to keep the deletion only when a file input number is + // given (which is more likely during pipeline development) and + // drop in any case when they are more than 100 + if( len<100 ) + delete << "rm -f ${Escape.path(stageName)}" // link them links << stageInputFile( storePath, stageName )