From 81a9294bbdc2b7ab699e96c6e1d46df248cc4364 Mon Sep 17 00:00:00 2001 From: Jake Tronge Date: Wed, 8 May 2024 09:17:33 -0600 Subject: [PATCH] Add beeflow:ScriptRequirement integration test Also refactors some of the other integration tests. --- beeflow/common/integration/utils.py | 7 ++++ beeflow/common/integration_test.py | 38 +++++++++++-------- ci/test_workflows/pre-post-script/input.yml | 1 + ci/test_workflows/pre-post-script/post.sh | 1 + ci/test_workflows/pre-post-script/pre.sh | 1 + .../pre-post-script/workflow.cwl | 33 ++++++++++++++++ 6 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 ci/test_workflows/pre-post-script/input.yml create mode 100644 ci/test_workflows/pre-post-script/post.sh create mode 100644 ci/test_workflows/pre-post-script/pre.sh create mode 100644 ci/test_workflows/pre-post-script/workflow.cwl diff --git a/beeflow/common/integration/utils.py b/beeflow/common/integration/utils.py index b6116724e..1c92213ad 100644 --- a/beeflow/common/integration/utils.py +++ b/beeflow/common/integration/utils.py @@ -250,3 +250,10 @@ def check_workflow_failed(workflow): """Ensure that the workflow completed in a Failed state.""" ci_assert(workflow.status == 'Archived/Failed', f'workflow did not fail as expected (final status: {workflow.status})') + + +def make_workflow_workdir(outer_workdir): + """Create a workdir for the workflow run output files.""" + workdir = os.path.join(outer_workdir, uuid.uuid4().hex) + os.makedirs(workdir) + return workdir diff --git a/beeflow/common/integration_test.py b/beeflow/common/integration_test.py index 9a8517faf..723d8069f 100644 --- a/beeflow/common/integration_test.py +++ b/beeflow/common/integration_test.py @@ -20,8 +20,7 @@ def copy_container(outer_workdir): # `beeflow:copyContainer` workflow container_path = f'/tmp/copy_container-{uuid.uuid4().hex}.tar.gz' container_name = 'copy-container' - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) container = utils.Container(container_name, generated_workflows.DOCKER_FILE_PATH, container_path) workflow_path = os.path.join(outer_workdir, f'bee-cc-workflow-{uuid.uuid4().hex}') @@ -94,8 +93,7 @@ def docker_file(outer_workdir): workflow_path = os.path.join(outer_workdir, f'bee-df-workflow-{uuid.uuid4().hex}') # Copy the Dockerfile to the workdir path os.makedirs(workflow_path) - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) shutil.copy(generated_workflows.DOCKER_FILE_PATH, os.path.join(workflow_path, 'Dockerfile')) container_name = 'docker_file_test' docker_requirement = { @@ -132,8 +130,7 @@ def docker_pull(outer_workdir): """Prepare, then check that the `dockerPull` option was successful.""" # `dockerPull` workflow workflow_path = os.path.join(outer_workdir, f'bee-dp-workflow-{uuid.uuid4().hex}') - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) container_name = generated_workflows.BASE_CONTAINER docker_requirement = { 'dockerPull': container_name, @@ -171,8 +168,7 @@ def multiple_workflows(outer_workdir): workflow_data = [] workflows = [] for i in range(3): - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) workflow_path = Path(outer_workdir, uuid.uuid4().hex) main_cwl, job_file = generated_workflows.simple_workflow(workflow_path, output_file) workflow = utils.Workflow(f'multi-workflow-{i}', workflow_path, @@ -197,8 +193,7 @@ def multiple_workflows(outer_workdir): @TEST_RUNNER.add() def build_failure(outer_workdir): """Test running a workflow with a bad container.""" - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) workflow = utils.Workflow('build-failure', 'ci/test_workflows/build-failure', main_cwl='workflow.cwl', job_file='input.yml', workdir=workdir, containers=[]) @@ -213,8 +208,7 @@ def build_failure(outer_workdir): @TEST_RUNNER.add() def dependent_tasks_fail(outer_workdir): """Test that dependent tasks don't run after a failure.""" - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) workflow = utils.Workflow('failure-dependent-tasks', 'ci/test_workflows/failure-dependent-tasks', main_cwl='workflow.cwl', job_file='input.yml', @@ -231,11 +225,24 @@ def dependent_tasks_fail(outer_workdir): f'task {task} did not get state DEP_FAIL as expected: {task_state}') +@TEST_RUNNER.add() +def pre_post_script(outer_workdir): + """Test that the beeflow:ScriptRequirement works.""" + workdir = utils.make_workflow_workdir(outer_workdir) + workflow = utils.Workflow('pre-post-script', 'ci/test_workflows/pre-post-script', + main_cwl='workflow.cwl', job_file='input.yml', + workdir=workdir, containers=[]) + yield [workflow] + utils.check_completed(workflow) + # Ensure files were touched by the pre and post scripts + utils.check_path_exists(Path(workdir, 'pre.txt')) + utils.check_path_exists(Path(workdir, 'post.txt')) + + @TEST_RUNNER.add(ignore=True) def checkpoint_restart(outer_workdir): """Test the clamr-ffmpeg checkpoint restart workflow.""" - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) workflow = utils.Workflow('checkpoint-restart', 'ci/test_workflows/clamr-wf-checkpoint', main_cwl='clamr_wf.cwl', job_file='clamr_job.yml', @@ -250,8 +257,7 @@ def checkpoint_restart(outer_workdir): @TEST_RUNNER.add(ignore=True) def checkpoint_restart_failure(outer_workdir): """Test a checkpoint restart workflow that continues past 'num_retries'.""" - workdir = os.path.join(outer_workdir, uuid.uuid4().hex) - os.makedirs(workdir) + workdir = utils.make_workflow_workdir(outer_workdir) workflow = utils.Workflow('checkpoint-too-long', 'ci/test_workflows/checkpoint-too-long', main_cwl='workflow.cwl', job_file='input.yml', diff --git a/ci/test_workflows/pre-post-script/input.yml b/ci/test_workflows/pre-post-script/input.yml new file mode 100644 index 000000000..eaf9dc0e3 --- /dev/null +++ b/ci/test_workflows/pre-post-script/input.yml @@ -0,0 +1 @@ +sleep_time: 1 diff --git a/ci/test_workflows/pre-post-script/post.sh b/ci/test_workflows/pre-post-script/post.sh new file mode 100644 index 000000000..bd45c97f9 --- /dev/null +++ b/ci/test_workflows/pre-post-script/post.sh @@ -0,0 +1 @@ +touch post.txt diff --git a/ci/test_workflows/pre-post-script/pre.sh b/ci/test_workflows/pre-post-script/pre.sh new file mode 100644 index 000000000..139e9084a --- /dev/null +++ b/ci/test_workflows/pre-post-script/pre.sh @@ -0,0 +1 @@ +touch pre.txt diff --git a/ci/test_workflows/pre-post-script/workflow.cwl b/ci/test_workflows/pre-post-script/workflow.cwl new file mode 100644 index 000000000..771bf6a10 --- /dev/null +++ b/ci/test_workflows/pre-post-script/workflow.cwl @@ -0,0 +1,33 @@ +class: Workflow +cwlVersion: v1.0 + +inputs: + sleep_time: int + +outputs: + step0_stdout: + type: File + outputSource: step0/step0_stdout + +steps: + step0: + run: + class: CommandLineTool + baseCommand: sleep + stdout: step0_stdout.txt + inputs: + sleep_time: + type: int + inputBinding: + position: 0 + outputs: + step0_stdout: + type: stdout + in: + sleep_time: sleep_time + out: [step0_stdout] + hints: + beeflow:ScriptRequirement: + enabled: true + pre_script: "pre.sh" + post_script: "post.sh"