Add integration test for pre/post scripts (beeflow:ScriptRequirement)

beeflow/common/integration/utils.py

@@ -250,3 +250,10 @@ def check_workflow_failed(workflow):
     """Ensure that the workflow completed in a Failed state."""
     ci_assert(workflow.status == 'Archived/Failed',
               f'workflow did not fail as expected (final status: {workflow.status})')
+
+
+def make_workflow_workdir(outer_workdir):
+    """Create a workdir for the workflow run output files."""
+    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
+    os.makedirs(workdir)
+    return workdir

beeflow/common/integration_test.py

@@ -20,8 +20,7 @@ def copy_container(outer_workdir):
     # `beeflow:copyContainer` workflow
     container_path = f'/tmp/copy_container-{uuid.uuid4().hex}.tar.gz'
     container_name = 'copy-container'
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     container = utils.Container(container_name, generated_workflows.DOCKER_FILE_PATH,
                                 container_path)
     workflow_path = os.path.join(outer_workdir, f'bee-cc-workflow-{uuid.uuid4().hex}')
@@ -94,8 +93,7 @@ def docker_file(outer_workdir):
     workflow_path = os.path.join(outer_workdir, f'bee-df-workflow-{uuid.uuid4().hex}')
     # Copy the Dockerfile to the workdir path
     os.makedirs(workflow_path)
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     shutil.copy(generated_workflows.DOCKER_FILE_PATH, os.path.join(workflow_path, 'Dockerfile'))
     container_name = 'docker_file_test'
     docker_requirement = {
@@ -132,8 +130,7 @@ def docker_pull(outer_workdir):
     """Prepare, then check that the `dockerPull` option was successful."""
     # `dockerPull` workflow
     workflow_path = os.path.join(outer_workdir, f'bee-dp-workflow-{uuid.uuid4().hex}')
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     container_name = generated_workflows.BASE_CONTAINER
     docker_requirement = {
         'dockerPull': container_name,
@@ -171,8 +168,7 @@ def multiple_workflows(outer_workdir):
     workflow_data = []
     workflows = []
     for i in range(3):
-        workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-        os.makedirs(workdir)
+        workdir = utils.make_workflow_workdir(outer_workdir)
         workflow_path = Path(outer_workdir, uuid.uuid4().hex)
         main_cwl, job_file = generated_workflows.simple_workflow(workflow_path, output_file)
         workflow = utils.Workflow(f'multi-workflow-{i}', workflow_path,
@@ -197,8 +193,7 @@ def multiple_workflows(outer_workdir):
 @TEST_RUNNER.add()
 def build_failure(outer_workdir):
     """Test running a workflow with a bad container."""
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     workflow = utils.Workflow('build-failure', 'ci/test_workflows/build-failure',
                               main_cwl='workflow.cwl', job_file='input.yml',
                               workdir=workdir, containers=[])
@@ -213,8 +208,7 @@ def build_failure(outer_workdir):
 @TEST_RUNNER.add()
 def dependent_tasks_fail(outer_workdir):
     """Test that dependent tasks don't run after a failure."""
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     workflow = utils.Workflow('failure-dependent-tasks',
                               'ci/test_workflows/failure-dependent-tasks',
                               main_cwl='workflow.cwl', job_file='input.yml',
@@ -231,11 +225,24 @@ def dependent_tasks_fail(outer_workdir):
                         f'task {task} did not get state DEP_FAIL as expected: {task_state}')
 
 
+@TEST_RUNNER.add()
+def pre_post_script(outer_workdir):
+    """Test that the beeflow:ScriptRequirement works."""
+    workdir = utils.make_workflow_workdir(outer_workdir)
+    workflow = utils.Workflow('pre-post-script', 'ci/test_workflows/pre-post-script',
+                              main_cwl='workflow.cwl', job_file='input.yml',
+                              workdir=workdir, containers=[])
+    yield [workflow]
+    utils.check_completed(workflow)
+    # Ensure files were touched by the pre and post scripts
+    utils.check_path_exists(Path(workdir, 'pre.txt'))
+    utils.check_path_exists(Path(workdir, 'post.txt'))
+
+
 @TEST_RUNNER.add(ignore=True)
 def checkpoint_restart(outer_workdir):
     """Test the clamr-ffmpeg checkpoint restart workflow."""
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     workflow = utils.Workflow('checkpoint-restart',
                               'ci/test_workflows/clamr-wf-checkpoint',
                               main_cwl='clamr_wf.cwl', job_file='clamr_job.yml',
@@ -250,8 +257,7 @@ def checkpoint_restart(outer_workdir):
 @TEST_RUNNER.add(ignore=True)
 def checkpoint_restart_failure(outer_workdir):
     """Test a checkpoint restart workflow that continues past 'num_retries'."""
-    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
-    os.makedirs(workdir)
+    workdir = utils.make_workflow_workdir(outer_workdir)
     workflow = utils.Workflow('checkpoint-too-long',
                               'ci/test_workflows/checkpoint-too-long',
                               main_cwl='workflow.cwl', job_file='input.yml',

ci/test_workflows/pre-post-script/input.yml

@@ -0,0 +1 @@
+sleep_time: 1

ci/test_workflows/pre-post-script/post.sh

@@ -0,0 +1 @@
+touch post.txt

ci/test_workflows/pre-post-script/pre.sh

@@ -0,0 +1 @@
+touch pre.txt

ci/test_workflows/pre-post-script/workflow.cwl

@@ -0,0 +1,33 @@
+class: Workflow
+cwlVersion: v1.0
+
+inputs:
+  sleep_time: int
+
+outputs:
+  step0_stdout:
+    type: File
+    outputSource: step0/step0_stdout
+
+steps:
+  step0:
+    run:
+      class: CommandLineTool
+      baseCommand: sleep
+      stdout: step0_stdout.txt
+      inputs:
+        sleep_time:
+          type: int
+          inputBinding:
+            position: 0
+      outputs:
+        step0_stdout:
+          type: stdout
+    in:
+      sleep_time: sleep_time
+    out: [step0_stdout]
+    hints:
+      beeflow:ScriptRequirement:
+        enabled: true
+        pre_script: "pre.sh"
+        post_script: "post.sh"