Potential flaky test or actual bug

[jaredoconnell]

Describe the bug

While running the CI, this test failed once:

❌ TestMultiDependencyDependOnClosedStepDeployment (20ms)
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Mimicking deployment of a plugin with image n/a for testing.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Plugin initialized.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Reading plugin schema...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	Starting ATP server in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Hello message read, ATP version 3.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Schema unserialization complete.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	ATP server execution finished in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Mimicking deployment of a plugin with image n/a for testing.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Plugin initialized.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Reading plugin schema...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	Starting ATP server in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Hello message read, ATP version 3.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Schema unserialization complete.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	ATP server execution finished in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Evaluating expression $.steps.wait_step.outputs...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Launching step wait_step...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Launching step cancelled_step...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Starting workflow execution...
          %% Mermaid markdown workflow
          flowchart LR
          %% Success path
          steps.cancelled_step.cancelled-->steps.cancelled_step.closed
          steps.cancelled_step.cancelled-->steps.cancelled_step.outputs
          steps.cancelled_step.closed-->steps.cancelled_step.closed.result
          steps.cancelled_step.closed.result-->outputs.closed
          steps.cancelled_step.deploy-->steps.cancelled_step.closed
          steps.cancelled_step.deploy-->steps.cancelled_step.starting
          steps.cancelled_step.disabled-->steps.cancelled_step.disabled.output
          steps.cancelled_step.enabling-->steps.cancelled_step.closed
          steps.cancelled_step.enabling-->steps.cancelled_step.disabled
          steps.cancelled_step.enabling-->steps.cancelled_step.enabling.resolved
          steps.cancelled_step.enabling-->steps.cancelled_step.starting
          steps.cancelled_step.outputs-->outputs.finished
          steps.cancelled_step.outputs-->steps.cancelled_step.outputs.cancelled_early
          steps.cancelled_step.outputs-->steps.cancelled_step.outputs.success
          steps.cancelled_step.outputs-->steps.cancelled_step.outputs.terminated_early
          steps.cancelled_step.running-->steps.cancelled_step.closed
          steps.cancelled_step.running-->steps.cancelled_step.outputs
          steps.cancelled_step.starting-->steps.cancelled_step.closed
          steps.cancelled_step.starting-->steps.cancelled_step.running
          steps.cancelled_step.starting-->steps.cancelled_step.starting.started
          steps.wait_step.cancelled-->steps.wait_step.closed
          steps.wait_step.cancelled-->steps.wait_step.outputs
          steps.wait_step.closed-->steps.wait_step.closed.result
          steps.wait_step.deploy-->steps.wait_step.closed
          steps.wait_step.deploy-->steps.wait_step.starting
          steps.wait_step.disabled-->steps.wait_step.disabled.output
          steps.wait_step.enabling-->steps.wait_step.closed
          steps.wait_step.enabling-->steps.wait_step.disabled
          steps.wait_step.enabling-->steps.wait_step.enabling.resolved
          steps.wait_step.enabling-->steps.wait_step.starting
          steps.wait_step.outputs-->steps.cancelled_step.cancelled
          steps.wait_step.outputs-->steps.wait_step.outputs.cancelled_early
          steps.wait_step.outputs-->steps.wait_step.outputs.success
          steps.wait_step.outputs-->steps.wait_step.outputs.terminated_early
          steps.wait_step.running-->steps.wait_step.closed
          steps.wait_step.running-->steps.wait_step.outputs
          steps.wait_step.starting-->steps.wait_step.closed
          steps.wait_step.starting-->steps.wait_step.running
          steps.wait_step.starting-->steps.wait_step.starting.started
          %% Error path
          steps.cancelled_step.cancelled-->steps.cancelled_step.crashed
          steps.cancelled_step.cancelled-->steps.cancelled_step.deploy_failed
          steps.cancelled_step.crashed-->steps.cancelled_step.crashed.error
          steps.cancelled_step.deploy-->steps.cancelled_step.deploy_failed
          steps.cancelled_step.deploy_failed-->steps.cancelled_step.deploy_failed.error
          steps.cancelled_step.enabling-->steps.cancelled_step.crashed
          steps.cancelled_step.running-->steps.cancelled_step.crashed
          steps.cancelled_step.starting-->steps.cancelled_step.crashed
          steps.wait_step.cancelled-->steps.wait_step.crashed
          steps.wait_step.cancelled-->steps.wait_step.deploy_failed
          steps.wait_step.crashed-->steps.wait_step.crashed.error
          steps.wait_step.deploy-->steps.wait_step.deploy_failed
          steps.wait_step.deploy_failed-->steps.wait_step.deploy_failed.error
          steps.wait_step.enabling-->steps.wait_step.crashed
          steps.wait_step.running-->steps.wait_step.crashed
          steps.wait_step.starting-->steps.wait_step.crashed
          %% Mermaid end
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 6 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.enabling with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.wait_step.enabling...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.enabling with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.cancelled_step.enabling...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.cancelled with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.wait_step.cancelled...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.deploy with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.wait_step.deploy...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.deploy with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.cancelled_step.deploy...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node input with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Deploying stage for step cancelled_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step cancelled_step to deploy...
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Mimicking deployment of a plugin with image n/a for testing.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Deploying stage for step wait_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step wait_step to deploy...
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Mimicking deployment of a plugin with image n/a for testing.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Plugin initialized.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Successfully deployed container with ID 'n/a' for step wait_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Checking to see if step wait_step/wait is enabled
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step wait_step to enabling...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.wait_step.deploy" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 0 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently waiting.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 1 waiting, 1 running, 0 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Step "wait_step" stage "disabled" declared that it will not produce an output (step enabled; cannot be disabled anymore)
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node wait_step.disabled.output in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node wait_step.disabled in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 1 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.disabled.output with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.wait_step.disabled.output with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Step wait_step/wait enablement state: true
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Starting stage for step wait_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step wait_step to starting and is waiting for input...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.wait_step.enabling" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving output node "steps.wait_step.enabling.resolved" in the DAG
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 3 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.enabling.resolved with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.starting with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.wait_step.starting...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.disabled with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.wait_step.disabled with type stepStage
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently waiting.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 1 waiting, 1 running, 0 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Reading plugin schema...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	Starting ATP server in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Hello message read, ATP version 3.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Schema unserialization complete.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=deployer	Plugin initialized.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Successfully deployed container with ID 'n/a' for step cancelled_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Checking to see if step cancelled_step/wait is enabled
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step cancelled_step to enabling...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.cancelled_step.deploy" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 0 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently waiting.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 1 waiting, 1 running, 0 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	Starting ATP server in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Step "cancelled_step" stage "disabled" declared that it will not produce an output (step enabled; cannot be disabled anymore)
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.disabled.output in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.disabled in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 1 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.disabled.output with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.cancelled_step.disabled.output with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Step cancelled_step/wait enablement state: true
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Starting stage for step cancelled_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step cancelled_step to starting and is waiting for input...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.cancelled_step.enabling" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving output node "steps.cancelled_step.enabling.resolved" in the DAG
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 3 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.enabling.resolved with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.starting with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.cancelled_step.starting...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.disabled with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.cancelled_step.disabled with type stepStage
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently waiting.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 1 waiting, 1 running, 0 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Reading plugin schema...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Executing plugin step wait_step/wait...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Preparing result channels for step with run ID "wait_step"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Step 'wait' started, waiting for response...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Running stage for step wait_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step wait_step to running and is waiting for input...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.wait_step.starting" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving output node "steps.wait_step.starting.started" in the DAG
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 2 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.starting.started with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.running with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 0 waiting, 2 running, 0 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Step with run ID 'wait_step' completed with output ID 'success'.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Sending results for run ID 'wait_step'
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	ATP server execution finished in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step wait_step to outputs and is waiting for input...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.wait_step.running" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 0 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 0 waiting, 2 running, 0 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Step wait_step completed with stage 'outputs', output 'success'...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	ATP signal loop done; channel closed
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Hello message read, ATP version 3.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.wait_step.outputs" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving output node "steps.wait_step.outputs.success" in the DAG
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node wait_step.outputs.terminated_early in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node wait_step.outputs.cancelled_early in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 4 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.outputs.success with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.outputs.cancelled_early with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.wait_step.outputs.cancelled_early with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.cancelled with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Evaluating expression $.steps.wait_step.outputs...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Providing stage input for steps.cancelled_step.cancelled...
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=plugin-provider	Cancelling step cancelled_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.wait_step.outputs.terminated_early with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.wait_step.outputs.terminated_early with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently finished.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 0 waiting, 1 running, 1 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Schema unserialization complete.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Running stage for step cancelled_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step cancelled_step to running and is waiting for input...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.cancelled_step.starting" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving output node "steps.cancelled_step.starting.started" in the DAG
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 2 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.starting.started with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.running with resolution status "waiting"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently finished.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 0 waiting, 1 running, 1 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=plugin-provider	Got step context done before step run complete. Sending cancellation signal. Waiting up to 0 milliseconds for result.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=plugin-provider	Cancelling step cancelled_step/wait
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=plugin-provider	Cancelled step cancelled_step/wait did not complete within the 0 millisecond time limit. Force closing container.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Stage change for step cancelled_step to crashed and is waiting for input...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.cancelled_step.running" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 0 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently finished.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently running.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 0 waiting, 1 running, 1 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=plugin-provider	Plugin step cancelled_step/wait run failed. step closed after timeout without result
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Step cancelled_step completed with stage 'crashed', output 'error'...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving node "steps.cancelled_step.crashed" in the DAG on stage complete
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Resolving output node "steps.cancelled_step.crashed.error" in the DAG
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 1 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.crashed.error with resolution status "resolved"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step cancelled_step is currently finished.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Step wait_step is currently finished.
      writer_target.go:15: 2024-09-13T14:49:34Z	info	source=workflow	There are currently 0 steps starting, 0 waiting, 0 running, 2 finished. HasReadyNodes: false
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=workflow	No running steps. Rechecking...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Step "cancelled_step" stage "outputs" declared that it will not produce an output (step closed after timeout without result)
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.outputs.success in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.outputs.cancelled_early in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.outputs.terminated_early in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.outputs in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 4 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.outputs.success with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.cancelled_step.outputs.success with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.outputs.cancelled_early with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.cancelled_step.outputs.cancelled_early with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.outputs.terminated_early with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.cancelled_step.outputs.terminated_early with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node outputs.finished with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Output node outputs.finished failed
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Step "cancelled_step" stage "closed" declared that it will not produce an output (step closed after timeout without result)
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.closed.result in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Will mark node cancelled_step.closed in the DAG as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Currently 2 DAG nodes are ready. Now processing them.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node outputs.closed with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Output node outputs.closed failed
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Processing step node steps.cancelled_step.closed.result with resolution status "unresolvable"
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Disregarding failed node steps.cancelled_step.closed.result with type stepStageOutput
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Executing plugin step cancelled_step/wait...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Preparing result channels for step with run ID "cancelled_step"
      writer_target.go:15: 2024-09-13T14:49:34Z	error	source=atp-client	Step 'wait' failed to write start work message: io: read/write on closed pipe
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=workflow	Workflow execution aborted with error: all outputs marked as unresolvable
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=executor	Workflow execution exited with error after context done
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Terminating all steps...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Terminating step wait_step...
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=plugin-provider	Step wait_step/wait force closed.
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=workflow	Terminating step cancelled_step...
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=deployer	ATP server execution finished in test deployer impl
          
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=atp-client	write called loop for run ID "cancelled_step" on done client; skipping receive loop
      writer_target.go:15: 2024-09-13T14:49:34Z	error	source=atp-client	ATP client for steps 'No running steps' failed to read or decode runtime message: io: read/write on closed pipe
      writer_target.go:15: 2024-09-13T14:49:34Z	debug	source=atp-client	Sending results for run ID 'cancelled_step'
      writer_target.go:15: 2024-09-13T14:49:34Z	warning	source=plugin-provider	Error while closing ATP client: client with step 'No running steps' failed to write client done message with error: io: read/write on closed pipe
      workflow_test.go:2572: MultiDependency DependOnClosedStep finished in 19 ms