From bb69e0a880b4d467951e648ec9977002d8b37dd4 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 10 Mar 2023 13:35:10 -0500 Subject: [PATCH 01/20] added list-outputs command --- src/cromshell/__main__.py | 2 + src/cromshell/list_outputs/__init__.py | 0 src/cromshell/list_outputs/command.py | 51 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 src/cromshell/list_outputs/__init__.py create mode 100644 src/cromshell/list_outputs/command.py diff --git a/src/cromshell/__main__.py b/src/cromshell/__main__.py index 7fcb1b7a..786fdc8b 100644 --- a/src/cromshell/__main__.py +++ b/src/cromshell/__main__.py @@ -10,6 +10,7 @@ from .alias import command as alias from .counts import command as counts from .list import command as list +from .list_outputs import command as list_outputs from .logs import command as logs from .metadata import command as metadata from .slim_metadata import command as slim_metadata @@ -147,6 +148,7 @@ def version(): main_entry.add_command(update_server.main) main_entry.add_command(timing.main) main_entry.add_command(list.main) +main_entry.add_command(list_outputs.main) if __name__ == "__main__": diff --git a/src/cromshell/list_outputs/__init__.py b/src/cromshell/list_outputs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py new file mode 100644 index 00000000..36f450d5 --- /dev/null +++ b/src/cromshell/list_outputs/command.py @@ -0,0 +1,51 @@ +import logging + +import click +import requests + +import cromshell.utilities.http_utils as http_utils +import cromshell.utilities.io_utils as io_utils +from cromshell.utilities import command_setup_utils + +LOGGER = logging.getLogger(__name__) + + +@click.command(name="list-outputs") +@click.argument("workflow_ids", required=True, nargs=-1) +@click.pass_obj +def main(config, workflow_ids): + """List workflow outputs.""" + + LOGGER.info("list-outputs") + + return_code = 0 + + for workflow_id in workflow_ids: + command_setup_utils.resolve_workflow_id_and_server( + workflow_id=workflow_id, cromshell_config=config + ) + + requests_out = requests.get( + f"{config.cromwell_api_workflow_id}/outputs", + timeout=config.requests_connect_timeout, + verify=config.requests_verify_certs, + headers=http_utils.generate_headers(config), + ) + + if requests_out.ok: + io_utils.pretty_print_json(format_json=requests_out.json()) + else: + return_code = 1 + + http_utils.check_http_request_status_code( + short_error_message="Failed to retrieve workflow outputs.", + response=requests_out, + # Raising exception is set false to allow + # command to retrieve outputs of remaining workflows. + raise_exception=False, + ) + + return return_code + + + From f2c7f0d7128bc5699228f16269402151dcf01f7c Mon Sep 17 00:00:00 2001 From: bshifaw Date: Mon, 13 Mar 2023 16:54:37 -0400 Subject: [PATCH 02/20] Added option to get workflow level outputs or task level outputs --- src/cromshell/list_outputs/command.py | 110 +++++++++++++++++++++----- 1 file changed, 92 insertions(+), 18 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index 36f450d5..49552dca 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -6,14 +6,22 @@ import cromshell.utilities.http_utils as http_utils import cromshell.utilities.io_utils as io_utils from cromshell.utilities import command_setup_utils +from cromshell.metadata import command as metadata_command LOGGER = logging.getLogger(__name__) @click.command(name="list-outputs") @click.argument("workflow_ids", required=True, nargs=-1) +@click.option( + "-d", + "--detailed", + is_flag=True, + default=False, + help="Get the output for a workflow at the task level", +) @click.pass_obj -def main(config, workflow_ids): +def main(config, workflow_ids, detailed): """List workflow outputs.""" LOGGER.info("list-outputs") @@ -25,27 +33,93 @@ def main(config, workflow_ids): workflow_id=workflow_id, cromshell_config=config ) - requests_out = requests.get( - f"{config.cromwell_api_workflow_id}/outputs", - timeout=config.requests_connect_timeout, - verify=config.requests_verify_certs, - headers=http_utils.generate_headers(config), + if not detailed: + io_utils.pretty_print_json(format_json=get_workflow_level_outputs(config)) + else: + print_task_level_outputs(get_task_level_outputs(config)) + + return return_code + + +def get_workflow_level_outputs(config) -> dict: + """Get the workflow level outputs from the workflow outputs""" + + requests_out = requests.get( + f"{config.cromwell_api_workflow_id}/outputs", + timeout=config.requests_connect_timeout, + verify=config.requests_verify_certs, + headers=http_utils.generate_headers(config), + ) + + if requests_out.ok: + return requests_out.json() + else: + + http_utils.check_http_request_status_code( + short_error_message="Failed to retrieve workflow outputs.", + response=requests_out, + # Raising exception is set false to allow + # command to retrieve outputs of remaining workflows. + raise_exception=False, ) - if requests_out.ok: - io_utils.pretty_print_json(format_json=requests_out.json()) - else: - return_code = 1 - http_utils.check_http_request_status_code( - short_error_message="Failed to retrieve workflow outputs.", - response=requests_out, - # Raising exception is set false to allow - # command to retrieve outputs of remaining workflows. - raise_exception=False, - ) +def get_task_level_outputs(config): + """Get the task level outputs from the workflow metadata - return return_code + Args: + config (dict): The cromshell config object + """ + # Get metadata + formatted_metadata_parameter = metadata_command.format_metadata_params( + list_of_keys=config.METADATA_KEYS_TO_OMIT, + exclude_keys=True, + expand_subworkflows=True, + ) + + workflow_metadata = metadata_command.get_workflow_metadata( + meta_params=formatted_metadata_parameter, + api_workflow_id=config.cromwell_api_workflow_id, + timeout=config.requests_connect_timeout, + verify_certs=config.requests_verify_certs, + headers=http_utils.generate_headers(config), + ) + + return get_outputs(workflow_metadata) + + +def get_outputs(workflow_metadata: dict): + """Get the outputs from the workflow metadata + + Args: + workflow_metadata (dict): The workflow metadata + """ + calls_metadata = workflow_metadata["calls"] + output_metadata = {} + extract_task_key = "outputs" + + for call, index_list in calls_metadata.items(): + if "subWorkflowMetadata" in calls_metadata[call][0]: + output_metadata[call] = [] + for scatter in calls_metadata[call]: + output_metadata[call].append( + get_outputs(scatter["subWorkflowMetadata"])) + else: + output_metadata[call] = [] + for index in index_list: + output_metadata[call].append(index.get(extract_task_key)) + + return output_metadata +def print_task_level_outputs(output_metadata: dict): + """Print the outputs from the workflow metadata + Args: + output_metadata (dict): The output metadata from the workflow + """ + for call, index_list in output_metadata.items(): + print(call) + for index in index_list: + if index is not None: + io_utils.pretty_print_json(format_json=index) From 3149dd20c97cfa16f7068550b68d081e525ce351 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Tue, 14 Mar 2023 10:25:09 -0400 Subject: [PATCH 03/20] Added option to print json summary and text --- src/cromshell/list_outputs/command.py | 73 +++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index 49552dca..ce9c1d84 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -20,8 +20,15 @@ default=False, help="Get the output for a workflow at the task level", ) +@click.option( + "-j", + "--json-summary", + is_flag=True, + default=False, + help="Print a json summary of the task outputs, including non file types.", +) @click.pass_obj -def main(config, workflow_ids, detailed): +def main(config, workflow_ids, detailed, json_summary): """List workflow outputs.""" LOGGER.info("list-outputs") @@ -34,9 +41,15 @@ def main(config, workflow_ids, detailed): ) if not detailed: - io_utils.pretty_print_json(format_json=get_workflow_level_outputs(config)) + if json_summary: + io_utils.pretty_print_json(format_json=get_workflow_level_outputs(config).get("outputs")) + else: + print_workflow_level_outputs(get_workflow_level_outputs(config)) else: - print_task_level_outputs(get_task_level_outputs(config)) + if json_summary: + io_utils.pretty_print_json(format_json=get_task_level_outputs(config)) + else: + print_task_level_outputs(get_task_level_outputs(config)) return return_code @@ -85,10 +98,10 @@ def get_task_level_outputs(config): headers=http_utils.generate_headers(config), ) - return get_outputs(workflow_metadata) + return filer_outputs_from_workflow_metadata(workflow_metadata) -def get_outputs(workflow_metadata: dict): +def filer_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: """Get the outputs from the workflow metadata Args: @@ -103,7 +116,7 @@ def get_outputs(workflow_metadata: dict): output_metadata[call] = [] for scatter in calls_metadata[call]: output_metadata[call].append( - get_outputs(scatter["subWorkflowMetadata"])) + filer_outputs_from_workflow_metadata(scatter["subWorkflowMetadata"])) else: output_metadata[call] = [] for index in index_list: @@ -112,14 +125,54 @@ def get_outputs(workflow_metadata: dict): return output_metadata -def print_task_level_outputs(output_metadata: dict): +def print_task_level_outputs(output_metadata: dict) -> None: """Print the outputs from the workflow metadata + output_metadata: {call_name:[index1{output_name: outputvalue}, index2{...}, ...], call_name:[], ...} Args: output_metadata (dict): The output metadata from the workflow """ for call, index_list in output_metadata.items(): print(call) - for index in index_list: - if index is not None: - io_utils.pretty_print_json(format_json=index) + for call_index in index_list: + if call_index is not None: + for task_output_name, task_output_value in call_index.items(): + if isinstance(task_output_value, str): + print_task_name_and_file(task_output_name, task_output_value) + elif isinstance(task_output_value, list): + for task_value in task_output_value: + print_task_name_and_file(task_output_name, task_value) + + +def print_workflow_level_outputs(workflow_outputs_json: dict) -> None: + """Print the workflow level outputs from the workflow outputs""" + workflow_outputs = workflow_outputs_json["outputs"] + + for workflow_output_name, workflow_output_value in workflow_outputs.items(): + if isinstance(workflow_output_value, str): + print_task_name_and_file(workflow_output_name, workflow_output_value, indent=False) + elif isinstance(workflow_output_value, list): + for task_value in workflow_output_value: + print_task_name_and_file(workflow_output_name, task_value, indent=False) + + +def print_task_name_and_file( + task_output_name: str, task_output_value: str, indent: bool = True +) -> None: + """Print the task name and the file name""" + + i = "\t" if indent else "" + + if isinstance(task_output_value, str): + if is_path_or_url_like(task_output_value): + print(f"{i}{task_output_name}: {task_output_value}") + + +def is_path_or_url_like(in_string: str) -> bool: + """Check if the string is a path or url""" + + if in_string.startswith("gs://") or in_string.startswith( + "/") or in_string.startswith("http://") or in_string.startswith("https://"): + return True + else: + return False From b85068b0fda29e338a916d1a4bbe45248cb5b2ef Mon Sep 17 00:00:00 2001 From: bshifaw Date: Tue, 14 Mar 2023 13:58:19 -0400 Subject: [PATCH 04/20] refactor functions --- src/cromshell/list_outputs/command.py | 66 ++++++++++++++++++--------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index ce9c1d84..4548320c 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -42,9 +42,14 @@ def main(config, workflow_ids, detailed, json_summary): if not detailed: if json_summary: - io_utils.pretty_print_json(format_json=get_workflow_level_outputs(config).get("outputs")) + io_utils.pretty_print_json( + format_json=get_workflow_level_outputs(config).get("outputs") + ) else: - print_workflow_level_outputs(get_workflow_level_outputs(config)) + print_output_metadata( + outputs_metadata=get_workflow_level_outputs(config).get("outputs"), + indent=False, + ) else: if json_summary: io_utils.pretty_print_json(format_json=get_task_level_outputs(config)) @@ -55,7 +60,10 @@ def main(config, workflow_ids, detailed, json_summary): def get_workflow_level_outputs(config) -> dict: - """Get the workflow level outputs from the workflow outputs""" + """Get the workflow level outputs from the workflow outputs + + Args: + config (dict): The cromshell config object""" requests_out = requests.get( f"{config.cromwell_api_workflow_id}/outputs", @@ -116,7 +124,9 @@ def filer_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: output_metadata[call] = [] for scatter in calls_metadata[call]: output_metadata[call].append( - filer_outputs_from_workflow_metadata(scatter["subWorkflowMetadata"])) + filer_outputs_from_workflow_metadata( + scatter["subWorkflowMetadata"]) + ) else: output_metadata[call] = [] for index in index_list: @@ -136,30 +146,38 @@ def print_task_level_outputs(output_metadata: dict) -> None: print(call) for call_index in index_list: if call_index is not None: - for task_output_name, task_output_value in call_index.items(): - if isinstance(task_output_value, str): - print_task_name_and_file(task_output_name, task_output_value) - elif isinstance(task_output_value, list): - for task_value in task_output_value: - print_task_name_and_file(task_output_name, task_value) + print_output_metadata(outputs_metadata=call_index, indent=True) + +def print_output_metadata(outputs_metadata: dict, indent: bool) -> None: + """Print the output metadata -def print_workflow_level_outputs(workflow_outputs_json: dict) -> None: - """Print the workflow level outputs from the workflow outputs""" - workflow_outputs = workflow_outputs_json["outputs"] + Args: + outputs_metadata (dict): The output metadata + indent (bool): Whether to indent the output + """ - for workflow_output_name, workflow_output_value in workflow_outputs.items(): - if isinstance(workflow_output_value, str): - print_task_name_and_file(workflow_output_name, workflow_output_value, indent=False) - elif isinstance(workflow_output_value, list): - for task_value in workflow_output_value: - print_task_name_and_file(workflow_output_name, task_value, indent=False) + for output_name, output_value in outputs_metadata.items(): + if isinstance(output_value, str): + print_output_name_and_file( + output_name, output_value, indent=indent + ) + elif isinstance(output_value, list): + for output_value_item in output_value: + print_output_name_and_file( + output_name, output_value_item, indent=indent + ) -def print_task_name_and_file( +def print_output_name_and_file( task_output_name: str, task_output_value: str, indent: bool = True ) -> None: - """Print the task name and the file name""" + """Print the task name and the file name + + Args: + task_output_name (str): The task output name + task_output_value (str): The task output value + indent (bool): Whether to indent the output""" i = "\t" if indent else "" @@ -169,7 +187,11 @@ def print_task_name_and_file( def is_path_or_url_like(in_string: str) -> bool: - """Check if the string is a path or url""" + """Check if the string is a path or url + + Args: + in_string (str): The string to check for path or url like-ness + """ if in_string.startswith("gs://") or in_string.startswith( "/") or in_string.startswith("http://") or in_string.startswith("https://"): From cc288c564fcaa63725a328030a4065fcbd0dc0fe Mon Sep 17 00:00:00 2001 From: bshifaw Date: Wed, 15 Mar 2023 08:43:21 -0400 Subject: [PATCH 05/20] unit test draft --- src/cromshell/list_outputs/command.py | 2 +- .../succeeded_helloworld.metadata.json | 245 ++++++++++++++++++ ...succeeded_helloworld.outputs.metadata.json | 1 + tests/unit/test_list_outputs.py | 71 +++++ 4 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 tests/metadata/succeeded_helloworld.metadata.json create mode 100644 tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json create mode 100644 tests/unit/test_list_outputs.py diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index 4548320c..ab7cbc68 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -85,7 +85,7 @@ def get_workflow_level_outputs(config) -> dict: ) -def get_task_level_outputs(config): +def get_task_level_outputs(config) -> dict: """Get the task level outputs from the workflow metadata Args: diff --git a/tests/metadata/succeeded_helloworld.metadata.json b/tests/metadata/succeeded_helloworld.metadata.json new file mode 100644 index 00000000..278d049c --- /dev/null +++ b/tests/metadata/succeeded_helloworld.metadata.json @@ -0,0 +1,245 @@ +{ + "actualWorkflowLanguage": "WDL", + "actualWorkflowLanguageVersion": "draft-2", + "calls": { + "HelloWorld.HelloWorldTask": [ + { + "attempt": 1, + "backend": "PAPIv2", + "backendLabels": { + "cromwell-workflow-id": "cromwell-9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b", + "wdl-task-name": "helloworldtask" + }, + "backendLogs": { + "log": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/HelloWorldTask.log" + }, + "backendStatus": "Success", + "callCaching": { + "allowResultReuse": true, + "effectiveCallCachingMode": "ReadAndWriteCache", + "hashes": { + "backend name": "85D6F63859525E464173387636E20324", + "command template": "265419AF4D334CAD1CC95E22CABFC8E5", + "input": { + "Boolean use_ssd": "68934A3E9455FA72420237EB05902327", + "Int command_mem": "051928341BE67DCBA03F0E04104D9047", + "Int default_boot_disk_size_gb": "9BF31C7FF062936A96D3C8BD1F8F2FF3", + "Int default_disk_space_gb": "F899139DF5E1059396431415E770C6DD", + "Int default_ram_mb": "BE767243CA8F574C740FB4C26CC6DCEB", + "Int machine_mem": "BE767243CA8F574C740FB4C26CC6DCEB", + "String docker": "1013E15CCD5A51F2B5A9F9DB9A13A756" + }, + "input count": "8F14E45FCEEA167A5A36DEDD4BEA2543", + "output count": "CFCD208495D565EF66E7DFF9F98764DA", + "runtime attribute": { + "continueOnReturnCode": "CFCD208495D565EF66E7DFF9F98764DA", + "docker": "D1FA3055EB3A898E0F62F288A334EB0A", + "failOnStderr": "68934A3E9455FA72420237EB05902327" + } + }, + "hit": false, + "result": "Cache Miss" + }, + "callRoot": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask", + "commandLine": " set -e\necho 'Hello World!'", + "compressedDockerSize": 4991278, + "dockerImageUsed": "frolvlad/alpine-bash@sha256:edac5ae03440fe8dcf3ff3410373c8dce56cca2915c74b5bf39afecff1693b28", + "end": "2022-10-21T19:12:53.198Z", + "executionEvents": [ + { + "description": "Worker \"google-pipelines-worker-6debdda87d7e05f880737046b3f87c2a\" assigned in \"us-east1-b\" on a \"custom-1-3072\" machine", + "endTime": "2022-10-21T19:10:54.271Z", + "startTime": "2022-10-21T19:10:15.348Z" + }, + { + "description": "Worker released", + "endTime": "2022-10-21T19:12:08.760Z", + "startTime": "2022-10-21T19:12:08.760Z" + }, + { + "description": "UpdatingJobStore", + "endTime": "2022-10-21T19:12:53.198Z", + "startTime": "2022-10-21T19:12:52.199Z" + }, + { + "description": "UserAction", + "endTime": "2022-10-21T19:11:54.971Z", + "startTime": "2022-10-21T19:11:50.783Z" + }, + { + "description": "CallCacheReading", + "endTime": "2022-10-21T19:09:48.639Z", + "startTime": "2022-10-21T19:09:48.634Z" + }, + { + "description": "Complete in GCE / Cromwell Poll Interval", + "endTime": "2022-10-21T19:12:52.016Z", + "startTime": "2022-10-21T19:12:08.760Z" + }, + { + "description": "Pending", + "endTime": "2022-10-21T19:09:48.049Z", + "startTime": "2022-10-21T19:09:48.048Z" + }, + { + "description": "WaitingForValueStore", + "endTime": "2022-10-21T19:09:48.266Z", + "startTime": "2022-10-21T19:09:48.266Z" + }, + { + "description": "RunningJob", + "endTime": "2022-10-21T19:10:03.560Z", + "startTime": "2022-10-21T19:09:48.639Z" + }, + { + "description": "ContainerSetup", + "endTime": "2022-10-21T19:11:30.236Z", + "startTime": "2022-10-21T19:11:26.183Z" + }, + { + "description": "Localization", + "endTime": "2022-10-21T19:11:50.783Z", + "startTime": "2022-10-21T19:11:31.132Z" + }, + { + "description": "Pulling \"frolvlad/alpine-bash@sha256:edac5ae03440fe8dcf3ff3410373c8dce56cca2915c74b5bf39afecff1693b28\"", + "endTime": "2022-10-21T19:11:26.183Z", + "startTime": "2022-10-21T19:11:23.952Z" + }, + { + "description": "waiting for quota", + "endTime": "2022-10-21T19:10:15.348Z", + "startTime": "2022-10-21T19:10:03.560Z" + }, + { + "description": "Pulling \"gcr.io/google.com/cloudsdktool/cloud-sdk:276.0.0-slim\"", + "endTime": "2022-10-21T19:11:23.952Z", + "startTime": "2022-10-21T19:10:54.271Z" + }, + { + "description": "PreparingJob", + "endTime": "2022-10-21T19:09:48.634Z", + "startTime": "2022-10-21T19:09:48.266Z" + }, + { + "description": "Background", + "endTime": "2022-10-21T19:11:30.760Z", + "startTime": "2022-10-21T19:11:30.427Z" + }, + { + "description": "Delocalization", + "endTime": "2022-10-21T19:12:08.760Z", + "startTime": "2022-10-21T19:11:54.971Z" + }, + { + "description": "RequestingExecutionToken", + "endTime": "2022-10-21T19:09:48.266Z", + "startTime": "2022-10-21T19:09:48.049Z" + }, + { + "description": "UpdatingCallCache", + "endTime": "2022-10-21T19:12:52.199Z", + "startTime": "2022-10-21T19:12:52.016Z" + } + ], + "executionStatus": "Done", + "inputs": { + "boot_disk_size_gb": null, + "command_mem": 2048, + "cpu": null, + "default_boot_disk_size_gb": 15, + "default_disk_space_gb": 100, + "default_ram_mb": 3072, + "disk_space_gb": null, + "docker": "frolvlad/alpine-bash", + "machine_mem": 3072, + "mem": null, + "preemptible_attempts": null, + "use_ssd": false + }, + "jes": { + "endpointUrl": "https://lifesciences.googleapis.com/", + "executionBucket": "gs://broad-dsp-lrma-cromwell-central", + "googleProject": "broad-dsp-lrma", + "instanceName": "google-pipelines-worker-6debdda87d7e05f880737046b3f87c2a", + "machineType": "custom-1-3072", + "zone": "us-east1-b" + }, + "jobId": "projects/602335226495/locations/us-central1/operations/7523289379686914908", + "labels": { + "cromwell-workflow-id": "cromwell-9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b", + "wdl-task-name": "HelloWorldTask" + }, + "outputs": { + "bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", + "bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", + "timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" + }, + "preemptible": false, + "returnCode": 0, + "runtimeAttributes": { + "bootDiskSizeGb": "15", + "continueOnReturnCode": "0", + "cpu": "1", + "cpuMin": "1", + "disks": "local-disk 100 HDD", + "docker": "frolvlad/alpine-bash", + "failOnStderr": "false", + "maxRetries": "0", + "memory": "3 GB", + "memoryMin": "2 GB", + "noAddress": "false", + "preemptible": "0", + "zones": "us-east1-b,us-east1-c,us-east1-d" + }, + "shardIndex": -1, + "start": "2022-10-21T19:09:48.048Z", + "stderr": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/stderr", + "stdout": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/stdout" + } + ] + }, + "end": "2022-10-21T19:12:53.940Z", + "id": "9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b", + "inputs": { + "HelloWorld.HelloWorldTask.default_boot_disk_size_gb": 15, + "HelloWorld.HelloWorldTask.default_disk_space_gb": 100, + "HelloWorld.HelloWorldTask.default_ram_mb": 3072, + "HelloWorld.HelloWorldTask.use_ssd": false, + "HelloWorld.boot_disk_size_gb": null, + "HelloWorld.cpu": null, + "HelloWorld.disk_space_gb": null, + "HelloWorld.docker": "frolvlad/alpine-bash", + "HelloWorld.mem": null, + "HelloWorld.preemptible_attempts": null + }, + "labels": { + "cromwell-workflow-id": "cromwell-9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b" + }, + "outputs": { + "bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", + "bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", + "timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" + + }, + "start": "2022-10-21T19:09:45.889Z", + "status": "Succeeded", + "submission": "2022-10-21T19:09:27.744Z", + "workflowName": "HelloWorld", + "workflowProcessingEvents": [ + { + "cromwellId": "cromid-dea8727", + "cromwellVersion": "63", + "description": "Finished", + "timestamp": "2022-10-21T19:12:53.940Z" + }, + { + "cromwellId": "cromid-dea8727", + "cromwellVersion": "63", + "description": "PickedUp", + "timestamp": "2022-10-21T19:09:45.888Z" + } + ], + "workflowRoot": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/" +} + diff --git a/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json b/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json new file mode 100644 index 00000000..a271555d --- /dev/null +++ b/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json @@ -0,0 +1 @@ +{'HelloWorld.HelloWorldTask': [{'bam_subset_file_index': 'gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai', 'bam_subset_file': 'gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam', 'timing_info': 'gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt'}]} \ No newline at end of file diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py new file mode 100644 index 00000000..539dc67f --- /dev/null +++ b/tests/unit/test_list_outputs.py @@ -0,0 +1,71 @@ +import json +import re + +import pytest + +from cromshell.list_outputs import command as list_outputs_command + + +class TestListOutputs: + """Test the execution list-outputs command functions""" + + @pytest.mark.parametrize( + "metadata_name, metadata_summary", + [ + [ + "failed_helloworld_metadata.json", + "counts/test_workflow_status_failed_helloword_metadata.txt", + ], + [ + "succeeded_workflow_slim_metadata.json", + "counts/test_workflow_status_succeded_workflow_slim_metadata.txt", + ], + ], + ) + def test_workflow_status( + self, mock_data_path, metadata_name, metadata_summary, ansi_escape, capsys + ): + """Note doesn't test for color of print out""" + + +# def get_workflow_level_outputs(config) -> dict: + + +# def test_get_task_level_outputs(config) -> dict: + + @pytest.mark.parametrize( + "workflow_metadata, outputs_metadata", + [ + [ + "succeeded_helloworld.metadata.json", + "list_outputs/succeeded_helloworld.outputs.metadata.json", + ], + ], + ) + def test_filer_outputs_from_workflow_metadata( + self, mock_data_path, tests_metadata_path, workflow_metadata, outputs_metadata, capsys + ): + with open(tests_metadata_path.joinpath(workflow_metadata), "r") as f: + workflow_metadata = json.load(f) + + with open(mock_data_path.joinpath(outputs_metadata), "r") as f: + outputs_metadata = f.read() + print("\n") + print(list_outputs_command.filer_outputs_from_workflow_metadata(workflow_metadata)) + + assert list_outputs_command.filer_outputs_from_workflow_metadata(workflow_metadata) == outputs_metadata + + +# +# +# def print_task_level_outputs(output_metadata: dict) -> None: +# +# +# def print_output_metadata(outputs_metadata: dict, indent: bool) -> None: +# +# +# def print_output_name_and_file( +# task_output_name: str, task_output_value: str, indent: bool = True +# ) -> None: +# +# def is_path_or_url_like(in_string: str) -> bool: From db2cefa1964d81541f5f71c899242b2b85d60262 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Wed, 15 Mar 2023 17:12:36 -0400 Subject: [PATCH 06/20] changed function names, added more unit tests --- src/cromshell/list_outputs/command.py | 24 +-- .../cromwell_output_api_example.json | 11 + .../helloworld_dict_of_outputs.json | 8 + .../helloworld_task_level_outputs.txt | 4 + .../helloworld_workflow_level_outputs.txt | 8 + .../print_file_like_value_in_dict_example.txt | 3 + ...e_like_value_in_dict_no_indent_example.txt | 3 + ...succeeded_helloworld.outputs.metadata.json | 10 +- tests/unit/test_list_outputs.py | 190 ++++++++++++++---- 9 files changed, 207 insertions(+), 54 deletions(-) create mode 100644 tests/unit/mock_data/list_outputs/cromwell_output_api_example.json create mode 100644 tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json create mode 100644 tests/unit/mock_data/list_outputs/helloworld_task_level_outputs.txt create mode 100644 tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt create mode 100644 tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_example.txt create mode 100644 tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_no_indent_example.txt diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index ab7cbc68..1d4a5fcf 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -25,11 +25,11 @@ "--json-summary", is_flag=True, default=False, - help="Print a json summary of the task outputs, including non file types.", + help="Print a json summary of outputs, including non-file types.", ) @click.pass_obj def main(config, workflow_ids, detailed, json_summary): - """List workflow outputs.""" + """List all output files produced by a workflow.""" LOGGER.info("list-outputs") @@ -46,7 +46,7 @@ def main(config, workflow_ids, detailed, json_summary): format_json=get_workflow_level_outputs(config).get("outputs") ) else: - print_output_metadata( + print_file_like_value_in_dict( outputs_metadata=get_workflow_level_outputs(config).get("outputs"), indent=False, ) @@ -146,11 +146,11 @@ def print_task_level_outputs(output_metadata: dict) -> None: print(call) for call_index in index_list: if call_index is not None: - print_output_metadata(outputs_metadata=call_index, indent=True) + print_file_like_value_in_dict(outputs_metadata=call_index, indent=True) -def print_output_metadata(outputs_metadata: dict, indent: bool) -> None: - """Print the output metadata +def print_file_like_value_in_dict(outputs_metadata: dict, indent: bool) -> None: + """Print the file like values in the output metadata dictionary Args: outputs_metadata (dict): The output metadata @@ -170,20 +170,20 @@ def print_output_metadata(outputs_metadata: dict, indent: bool) -> None: def print_output_name_and_file( - task_output_name: str, task_output_value: str, indent: bool = True + output_name: str, output_value: str, indent: bool = True ) -> None: """Print the task name and the file name Args: - task_output_name (str): The task output name - task_output_value (str): The task output value + output_name (str): The task output name + output_value (str): The task output value indent (bool): Whether to indent the output""" i = "\t" if indent else "" - if isinstance(task_output_value, str): - if is_path_or_url_like(task_output_value): - print(f"{i}{task_output_name}: {task_output_value}") + if isinstance(output_value, str): + if is_path_or_url_like(output_value): + print(f"{i}{output_name}: {output_value}") def is_path_or_url_like(in_string: str) -> bool: diff --git a/tests/unit/mock_data/list_outputs/cromwell_output_api_example.json b/tests/unit/mock_data/list_outputs/cromwell_output_api_example.json new file mode 100644 index 00000000..96425b83 --- /dev/null +++ b/tests/unit/mock_data/list_outputs/cromwell_output_api_example.json @@ -0,0 +1,11 @@ +{ + "id": "9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b", + "output": { + "HelloWorld.analysis_ready_bam_size": 0, + "HelloWorld.analysis_ready_bam_size_in_gb": 132, + "HelloWorld.analysis_ready_bam_name": "NA12878.hg38.bam", + "HelloWorld.HelloWorldTask.bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", + "HelloWorld.HelloWorldTask.bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", + "HelloWorld.HelloWorldTask.timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" + } +} diff --git a/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json b/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json new file mode 100644 index 00000000..85fd632e --- /dev/null +++ b/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json @@ -0,0 +1,8 @@ +{ + "HelloWorld.analysis_ready_bam_size": 0, + "HelloWorld.analysis_ready_bam_size_in_gb": 132, + "HelloWorld.analysis_ready_bam_name": "NA12878.hg38.bam", + "HelloWorld.HelloWorldTask.bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", + "HelloWorld.HelloWorldTask.bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", + "HelloWorld.HelloWorldTask.timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" +} \ No newline at end of file diff --git a/tests/unit/mock_data/list_outputs/helloworld_task_level_outputs.txt b/tests/unit/mock_data/list_outputs/helloworld_task_level_outputs.txt new file mode 100644 index 00000000..0b3a4568 --- /dev/null +++ b/tests/unit/mock_data/list_outputs/helloworld_task_level_outputs.txt @@ -0,0 +1,4 @@ +HelloWorld.HelloWorldTask + bam_subset_file_index: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai + bam_subset_file: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam + timing_info: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt diff --git a/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt b/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt new file mode 100644 index 00000000..85fd632e --- /dev/null +++ b/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt @@ -0,0 +1,8 @@ +{ + "HelloWorld.analysis_ready_bam_size": 0, + "HelloWorld.analysis_ready_bam_size_in_gb": 132, + "HelloWorld.analysis_ready_bam_name": "NA12878.hg38.bam", + "HelloWorld.HelloWorldTask.bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", + "HelloWorld.HelloWorldTask.bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", + "HelloWorld.HelloWorldTask.timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" +} \ No newline at end of file diff --git a/tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_example.txt b/tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_example.txt new file mode 100644 index 00000000..fddd77d2 --- /dev/null +++ b/tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_example.txt @@ -0,0 +1,3 @@ + HelloWorld.HelloWorldTask.bam_subset_file_index: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai + HelloWorld.HelloWorldTask.bam_subset_file: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam + HelloWorld.HelloWorldTask.timing_info: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt diff --git a/tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_no_indent_example.txt b/tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_no_indent_example.txt new file mode 100644 index 00000000..86247ddf --- /dev/null +++ b/tests/unit/mock_data/list_outputs/print_file_like_value_in_dict_no_indent_example.txt @@ -0,0 +1,3 @@ +HelloWorld.HelloWorldTask.bam_subset_file_index: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai +HelloWorld.HelloWorldTask.bam_subset_file: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam +HelloWorld.HelloWorldTask.timing_info: gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt diff --git a/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json b/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json index a271555d..6f97c485 100644 --- a/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json +++ b/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json @@ -1 +1,9 @@ -{'HelloWorld.HelloWorldTask': [{'bam_subset_file_index': 'gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai', 'bam_subset_file': 'gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam', 'timing_info': 'gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt'}]} \ No newline at end of file +{ + "HelloWorld.HelloWorldTask": [ + { + "bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", + "bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", + "timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" + } + ] +} \ No newline at end of file diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index 539dc67f..e26ed117 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -1,5 +1,4 @@ import json -import re import pytest @@ -9,63 +8,172 @@ class TestListOutputs: """Test the execution list-outputs command functions""" + +# def get_workflow_level_outputs(config) -> dict: + + +# def test_get_task_level_outputs(config) -> dict: + @pytest.mark.parametrize( - "metadata_name, metadata_summary", + "workflow_metadata_file, outputs_metadata_file_path", [ [ - "failed_helloworld_metadata.json", - "counts/test_workflow_status_failed_helloword_metadata.txt", + "succeeded_helloworld.metadata.json", + "list_outputs/succeeded_helloworld.outputs.metadata.json", ], + ], + ) + def test_filer_outputs_from_workflow_metadata( + self, + mock_data_path, + tests_metadata_path, + workflow_metadata_file, + outputs_metadata_file_path, + ): + with open(tests_metadata_path.joinpath(workflow_metadata_file), "r") as f: + workflow_metadata = json.load(f) + + with open(mock_data_path.joinpath(outputs_metadata_file_path), "r") as f: + outputs_metadata = json.load(f) + + assert list_outputs_command.filer_outputs_from_workflow_metadata( + workflow_metadata + ) == outputs_metadata + + + @pytest.mark.parametrize( + "outputs_metadata_file_path, expected_task_level_outputs_file_path", + [ [ - "succeeded_workflow_slim_metadata.json", - "counts/test_workflow_status_succeded_workflow_slim_metadata.txt", + "list_outputs/succeeded_helloworld.outputs.metadata.json", + "list_outputs/helloworld_task_level_outputs.txt", ], ], ) - def test_workflow_status( - self, mock_data_path, metadata_name, metadata_summary, ansi_escape, capsys - ): - """Note doesn't test for color of print out""" + def test_print_task_level_outputs( + self, + outputs_metadata_file_path: dict, + mock_data_path, + expected_task_level_outputs_file_path, + capsys, + ) -> None: + """Test the print_task_level_outputs function""" + with open(mock_data_path.joinpath(outputs_metadata_file_path), "r") as f: + outputs_metadata = json.load(f) + with open(mock_data_path.joinpath(expected_task_level_outputs_file_path), "r") as f: + expected_task_level_outputs = f.read() -# def get_workflow_level_outputs(config) -> dict: + list_outputs_command.print_task_level_outputs(outputs_metadata) + captured = capsys.readouterr() + assert captured.out == expected_task_level_outputs -# def test_get_task_level_outputs(config) -> dict: @pytest.mark.parametrize( - "workflow_metadata, outputs_metadata", + "outputs_api_example_file, expected_workflow_level_outputs_file_path, indent", [ [ - "succeeded_helloworld.metadata.json", - "list_outputs/succeeded_helloworld.outputs.metadata.json", + "list_outputs/helloworld_dict_of_outputs.json", + "list_outputs/print_file_like_value_in_dict_example.txt", + True, + ], + [ + "list_outputs/helloworld_dict_of_outputs.json", + "list_outputs/print_file_like_value_in_dict_no_indent_example.txt", + False, ], ], ) - def test_filer_outputs_from_workflow_metadata( - self, mock_data_path, tests_metadata_path, workflow_metadata, outputs_metadata, capsys - ): - with open(tests_metadata_path.joinpath(workflow_metadata), "r") as f: - workflow_metadata = json.load(f) + def test_print_output_metadata( + self, + outputs_api_example_file, + tests_metadata_path, + mock_data_path, + expected_workflow_level_outputs_file_path, + indent, + capsys, + ) -> None: - with open(mock_data_path.joinpath(outputs_metadata), "r") as f: - outputs_metadata = f.read() - print("\n") - print(list_outputs_command.filer_outputs_from_workflow_metadata(workflow_metadata)) - - assert list_outputs_command.filer_outputs_from_workflow_metadata(workflow_metadata) == outputs_metadata - - -# -# -# def print_task_level_outputs(output_metadata: dict) -> None: -# -# -# def print_output_metadata(outputs_metadata: dict, indent: bool) -> None: -# -# -# def print_output_name_and_file( -# task_output_name: str, task_output_value: str, indent: bool = True -# ) -> None: -# -# def is_path_or_url_like(in_string: str) -> bool: + with open(mock_data_path.joinpath(outputs_api_example_file), "r") as f: + outputs_metadata = json.load(f) + with open( + mock_data_path.joinpath(expected_workflow_level_outputs_file_path), "r" + ) as f: + expected_workflow_level_outputs = f.read() + + list_outputs_command.print_file_like_value_in_dict( + outputs_metadata, indent=indent + ) + + captured = capsys.readouterr() + assert captured.out == expected_workflow_level_outputs + + @pytest.mark.parametrize( + "output_name, output_value, indent, expected_function_print", + [ + [ + "task_name", + "/taskoutputfile", + True, + "\ttask_name: /taskoutputfile\n", + ], + [ + "task_name", + "taskoutputfile", + True, + "", + ], + [ + "task_name", + "gs://taskoutputfile", + True, + "\ttask_name: gs://taskoutputfile\n", + ], + ], + ) + def test_print_output_name_and_file( + self, + output_name, + output_value, + indent, + expected_function_print, + capsys, + ) -> None: + + list_outputs_command.print_output_name_and_file( + output_name=output_name, + output_value=output_value, + indent=indent, + ) + + captured = capsys.readouterr() + assert captured.out == expected_function_print + + @pytest.mark.parametrize( + "value, expected_bool", + [ + [ + "task_value", + False, + ], + [ + "/task_value", + True, + ], + [ + "gs://task_value", + True, + ], + [ + "task_value/", + False, + ], + [ + "http://task_value", + True, + ], + ], + ) + def test_is_path_or_url_like(self, value, expected_bool): + assert list_outputs_command.is_path_or_url_like(value) == expected_bool From 565a5b255bc5c04a5233e2ccef3f29c442243032 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 16 Mar 2023 11:38:16 -0400 Subject: [PATCH 07/20] Lint fixes --- src/cromshell/list_outputs/command.py | 32 +++++++++++++-------------- tests/unit/test_list_outputs.py | 24 +++++++++----------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index 1d4a5fcf..4bc79d7e 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -5,8 +5,8 @@ import cromshell.utilities.http_utils as http_utils import cromshell.utilities.io_utils as io_utils -from cromshell.utilities import command_setup_utils from cromshell.metadata import command as metadata_command +from cromshell.utilities import command_setup_utils LOGGER = logging.getLogger(__name__) @@ -63,7 +63,8 @@ def get_workflow_level_outputs(config) -> dict: """Get the workflow level outputs from the workflow outputs Args: - config (dict): The cromshell config object""" + config (dict): The cromshell config object + """ requests_out = requests.get( f"{config.cromwell_api_workflow_id}/outputs", @@ -75,7 +76,6 @@ def get_workflow_level_outputs(config) -> dict: if requests_out.ok: return requests_out.json() else: - http_utils.check_http_request_status_code( short_error_message="Failed to retrieve workflow outputs.", response=requests_out, @@ -90,7 +90,7 @@ def get_task_level_outputs(config) -> dict: Args: config (dict): The cromshell config object - """ + """ # Get metadata formatted_metadata_parameter = metadata_command.format_metadata_params( list_of_keys=config.METADATA_KEYS_TO_OMIT, @@ -114,7 +114,7 @@ def filer_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: Args: workflow_metadata (dict): The workflow metadata - """ + """ calls_metadata = workflow_metadata["calls"] output_metadata = {} extract_task_key = "outputs" @@ -124,8 +124,7 @@ def filer_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: output_metadata[call] = [] for scatter in calls_metadata[call]: output_metadata[call].append( - filer_outputs_from_workflow_metadata( - scatter["subWorkflowMetadata"]) + filer_outputs_from_workflow_metadata(scatter["subWorkflowMetadata"]) ) else: output_metadata[call] = [] @@ -141,7 +140,7 @@ def print_task_level_outputs(output_metadata: dict) -> None: Args: output_metadata (dict): The output metadata from the workflow - """ + """ for call, index_list in output_metadata.items(): print(call) for call_index in index_list: @@ -155,13 +154,11 @@ def print_file_like_value_in_dict(outputs_metadata: dict, indent: bool) -> None: Args: outputs_metadata (dict): The output metadata indent (bool): Whether to indent the output - """ + """ for output_name, output_value in outputs_metadata.items(): if isinstance(output_value, str): - print_output_name_and_file( - output_name, output_value, indent=indent - ) + print_output_name_and_file(output_name, output_value, indent=indent) elif isinstance(output_value, list): for output_value_item in output_value: print_output_name_and_file( @@ -191,10 +188,13 @@ def is_path_or_url_like(in_string: str) -> bool: Args: in_string (str): The string to check for path or url like-ness - """ - - if in_string.startswith("gs://") or in_string.startswith( - "/") or in_string.startswith("http://") or in_string.startswith("https://"): + """ + if ( + in_string.startswith("gs://") + or in_string.startswith("/") + or in_string.startswith("http://") + or in_string.startswith("https://") + ): return True else: return False diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index e26ed117..c7dca44b 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -8,11 +8,8 @@ class TestListOutputs: """Test the execution list-outputs command functions""" - -# def get_workflow_level_outputs(config) -> dict: - - -# def test_get_task_level_outputs(config) -> dict: + # def get_workflow_level_outputs(config) -> dict: + # def test_get_task_level_outputs(config) -> dict: @pytest.mark.parametrize( "workflow_metadata_file, outputs_metadata_file_path", @@ -36,10 +33,10 @@ def test_filer_outputs_from_workflow_metadata( with open(mock_data_path.joinpath(outputs_metadata_file_path), "r") as f: outputs_metadata = json.load(f) - assert list_outputs_command.filer_outputs_from_workflow_metadata( - workflow_metadata - ) == outputs_metadata - + assert ( + list_outputs_command.filer_outputs_from_workflow_metadata(workflow_metadata) + == outputs_metadata + ) @pytest.mark.parametrize( "outputs_metadata_file_path, expected_task_level_outputs_file_path", @@ -61,7 +58,9 @@ def test_print_task_level_outputs( with open(mock_data_path.joinpath(outputs_metadata_file_path), "r") as f: outputs_metadata = json.load(f) - with open(mock_data_path.joinpath(expected_task_level_outputs_file_path), "r") as f: + with open( + mock_data_path.joinpath(expected_task_level_outputs_file_path), "r" + ) as f: expected_task_level_outputs = f.read() list_outputs_command.print_task_level_outputs(outputs_metadata) @@ -69,7 +68,6 @@ def test_print_task_level_outputs( captured = capsys.readouterr() assert captured.out == expected_task_level_outputs - @pytest.mark.parametrize( "outputs_api_example_file, expected_workflow_level_outputs_file_path, indent", [ @@ -94,11 +92,10 @@ def test_print_output_metadata( indent, capsys, ) -> None: - with open(mock_data_path.joinpath(outputs_api_example_file), "r") as f: outputs_metadata = json.load(f) with open( - mock_data_path.joinpath(expected_workflow_level_outputs_file_path), "r" + mock_data_path.joinpath(expected_workflow_level_outputs_file_path), "r" ) as f: expected_workflow_level_outputs = f.read() @@ -140,7 +137,6 @@ def test_print_output_name_and_file( expected_function_print, capsys, ) -> None: - list_outputs_command.print_output_name_and_file( output_name=output_name, output_value=output_value, From d02d6dbbf0260bebe8795dd60b49a7d8c977e60f Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 16 Mar 2023 11:41:42 -0400 Subject: [PATCH 08/20] adding end of line --- .../unit/mock_data/list_outputs/helloworld_dict_of_outputs.json | 2 +- .../list_outputs/helloworld_workflow_level_outputs.txt | 2 +- .../list_outputs/succeeded_helloworld.outputs.metadata.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json b/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json index 85fd632e..25843e38 100644 --- a/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json +++ b/tests/unit/mock_data/list_outputs/helloworld_dict_of_outputs.json @@ -5,4 +5,4 @@ "HelloWorld.HelloWorldTask.bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", "HelloWorld.HelloWorldTask.bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", "HelloWorld.HelloWorldTask.timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" -} \ No newline at end of file +} diff --git a/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt b/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt index 85fd632e..25843e38 100644 --- a/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt +++ b/tests/unit/mock_data/list_outputs/helloworld_workflow_level_outputs.txt @@ -5,4 +5,4 @@ "HelloWorld.HelloWorldTask.bam_subset_file_index": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bai", "HelloWorld.HelloWorldTask.bam_subset_file": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.chr2.bam", "HelloWorld.HelloWorldTask.timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" -} \ No newline at end of file +} diff --git a/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json b/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json index 6f97c485..0bb3b079 100644 --- a/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json +++ b/tests/unit/mock_data/list_outputs/succeeded_helloworld.outputs.metadata.json @@ -6,4 +6,4 @@ "timing_info": "gs://broad-dsp-lrma-cromwell-central/HelloWorld/9ee4aa2e-7ac5-4c61-88b2-88a4d10f168b/call-HelloWorldTask/shard-1/m64020_190210_035026.subreads.ccs.uncorrected.aligned.merged.timingInformation.txt" } ] -} \ No newline at end of file +} From dce9ee52b8b4960a3f664d8608ffbdc3289004ff Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 17 Mar 2023 12:19:24 -0400 Subject: [PATCH 09/20] add integration test_list_outputs.py draft --- tests/integration/test_list_outputs.py | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/integration/test_list_outputs.py diff --git a/tests/integration/test_list_outputs.py b/tests/integration/test_list_outputs.py new file mode 100644 index 00000000..3c2d31fe --- /dev/null +++ b/tests/integration/test_list_outputs.py @@ -0,0 +1,72 @@ +from pathlib import Path + +import pytest + +from tests.integration import utility_test_functions + +workflows_path = Path(__file__).parents[1].joinpath("workflows/") + + +class TestListOutputs: + @pytest.mark.parametrize( + "wdl, json_file, should_fail, call_summary, workflow_state", + [ + ( + "tests/workflows/helloWorld.wdl", + "tests/workflows/helloWorld.json", + False, + "\tHelloWorld.HelloWorldTask\t0 Running, 1 Done, 0 Preempted, 0 Failed", + "Succeeded", + ), + # ( + # "tests/workflows/helloWorldFail.wdl", + # "tests/workflows/helloWorld.json", + # True, + # "\tHelloWorld.HelloWorldTask\t0 Running, 0 Done, 0 Preempted, 1 Failed", + # "Failed", + # ), + ], + ) + def test_counts( + self, + local_cromwell_url: str, + wdl: str, + json_file: str, + should_fail: bool, + call_summary: str, + workflow_state: str, + ansi_escape, + ): + # submit workflow + test_workflow_id = utility_test_functions.submit_workflow( + local_cromwell_url=local_cromwell_url, + wdl=wdl, + json_file=json_file, + exit_code=0, + ) + + utility_test_functions.wait_for_workflow_completion( + test_workflow_id=test_workflow_id + ) + + # run list-outputs + status_result = utility_test_functions.run_cromshell_command( + command=["list-outputs", test_workflow_id], + exit_code=0, + ) + + status_result_per_line = status_result.stdout.split("\n") + + print("Print workflow list-outputs results:") + for line in status_result_per_line: + print(line) + + # The workflows being used here will only generate 2 lines from the counts + # command, but if testing more complicated workflows the second_line will + # need to be asserted in a different manner to include all the calls within + # the workflow being tested. + first_line = ansi_escape.sub("", status_result_per_line[0]) + second_line = ansi_escape.sub("", status_result_per_line[1]) + + assert first_line == test_workflow_id + "\t" + workflow_state + assert second_line == call_summary From 8f77751d7b384a45b0b751f7dba0db5c4be715e6 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 23 Mar 2023 18:34:02 -0400 Subject: [PATCH 10/20] added output to helloWorld.wdl --- tests/workflows/helloWorld.wdl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/workflows/helloWorld.wdl b/tests/workflows/helloWorld.wdl index 6ea59c64..a0eb63f1 100644 --- a/tests/workflows/helloWorld.wdl +++ b/tests/workflows/helloWorld.wdl @@ -32,6 +32,7 @@ workflow HelloWorld { } output { + File output_file = HelloWorldTask.output_file } } @@ -89,6 +90,7 @@ task HelloWorldTask { # ------------------------------------------------ # Outputs: output { + File output_file = stdout() } } From ce3297b16b69277edaadee6aee81f8af87c4d460 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 24 Mar 2023 10:38:40 -0400 Subject: [PATCH 11/20] added options to utility_test_functions.py run cromshell function --- tests/integration/utility_test_functions.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/utility_test_functions.py b/tests/integration/utility_test_functions.py index 7da27a82..82eec8f3 100644 --- a/tests/integration/utility_test_functions.py +++ b/tests/integration/utility_test_functions.py @@ -9,10 +9,11 @@ from cromshell.utilities import cromshellconfig -def run_cromshell_command(command: list, exit_code: int): +def run_cromshell_command(command: list, exit_code: int, options: list = None): """ Run cromshell alias using CliRunner and assert job is successful + :param options: :param command: The subcommand, options, and arguments in list form e.g. [ "alias", @@ -24,12 +25,18 @@ def run_cromshell_command(command: list, exit_code: int): :return: results from execution """ + command_with_options = command.copy() + if options: + for item in options: + command_with_options.insert(1, item) + runner = CliRunner(mix_stderr=False) # The absolute path will be passed to the invoke command because # the test is being run in temp directory created by CliRunner. with runner.isolated_filesystem(): - result = runner.invoke(cromshell, command) + result = runner.invoke(cromshell, command_with_options) assert result.exit_code == exit_code, ( + f"\nCOMMAND:\n{command_with_options}" f"\nSTDOUT:\n{result.stdout}" f"\nSTDERR:\n{result.stderr}" f"\nExceptions:\n{result.exception}" From f29b545b94d8176a9af8357aa4d7b7711a29d962 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 24 Mar 2023 10:39:23 -0400 Subject: [PATCH 12/20] working version of list-outputs integration test --- tests/integration/test_list_outputs.py | 75 +++++++++++++++++--------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/tests/integration/test_list_outputs.py b/tests/integration/test_list_outputs.py index 3c2d31fe..b21c8a10 100644 --- a/tests/integration/test_list_outputs.py +++ b/tests/integration/test_list_outputs.py @@ -9,32 +9,62 @@ class TestListOutputs: @pytest.mark.parametrize( - "wdl, json_file, should_fail, call_summary, workflow_state", + "wdl, json_file, options, output_template", [ ( "tests/workflows/helloWorld.wdl", "tests/workflows/helloWorld.json", - False, - "\tHelloWorld.HelloWorldTask\t0 Running, 1 Done, 0 Preempted, 0 Failed", - "Succeeded", + None, + [ + "HelloWorld.output_file: /cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout", + "" + ], + ), + ( + "tests/workflows/helloWorld.wdl", + "tests/workflows/helloWorld.json", + ["-d"], + [ + "HelloWorld.HelloWorldTask", + "\toutput_file: /cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout", + "" + ], + ), + ( + "tests/workflows/helloWorld.wdl", + "tests/workflows/helloWorld.json", + ["-j"], + [ + "{", + ' "HelloWorld.output_file": "/cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout"', + "}", + "" + ], + ), + ( + "tests/workflows/helloWorld.wdl", + "tests/workflows/helloWorld.json", + ["-j", "-d"], + [ + "{", + ' "HelloWorld.HelloWorldTask": [', + " {", + ' "output_file": "/cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout"', + " }", + " ]", + "}", + "" + ], ), - # ( - # "tests/workflows/helloWorldFail.wdl", - # "tests/workflows/helloWorld.json", - # True, - # "\tHelloWorld.HelloWorldTask\t0 Running, 0 Done, 0 Preempted, 1 Failed", - # "Failed", - # ), ], ) - def test_counts( + def test_list_outputs( self, local_cromwell_url: str, wdl: str, json_file: str, - should_fail: bool, - call_summary: str, - workflow_state: str, + options: list, + output_template: list, ansi_escape, ): # submit workflow @@ -53,20 +83,17 @@ def test_counts( status_result = utility_test_functions.run_cromshell_command( command=["list-outputs", test_workflow_id], exit_code=0, + options=options, ) status_result_per_line = status_result.stdout.split("\n") + workflow_outputs = [ + sub.replace('', test_workflow_id) for sub in output_template + ] + print("Print workflow list-outputs results:") for line in status_result_per_line: print(line) - # The workflows being used here will only generate 2 lines from the counts - # command, but if testing more complicated workflows the second_line will - # need to be asserted in a different manner to include all the calls within - # the workflow being tested. - first_line = ansi_escape.sub("", status_result_per_line[0]) - second_line = ansi_escape.sub("", status_result_per_line[1]) - - assert first_line == test_workflow_id + "\t" + workflow_state - assert second_line == call_summary + assert status_result_per_line == workflow_outputs From d6d95c3ba4f22df880fc357351405f61520fd7dc Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 24 Mar 2023 10:51:57 -0400 Subject: [PATCH 13/20] lint fix --- tests/integration/test_list_outputs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_list_outputs.py b/tests/integration/test_list_outputs.py index b21c8a10..754ae545 100644 --- a/tests/integration/test_list_outputs.py +++ b/tests/integration/test_list_outputs.py @@ -17,7 +17,7 @@ class TestListOutputs: None, [ "HelloWorld.output_file: /cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout", - "" + "", ], ), ( @@ -27,7 +27,7 @@ class TestListOutputs: [ "HelloWorld.HelloWorldTask", "\toutput_file: /cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout", - "" + "", ], ), ( @@ -38,7 +38,7 @@ class TestListOutputs: "{", ' "HelloWorld.output_file": "/cromwell-executions/HelloWorld//call-HelloWorldTask/execution/stdout"', "}", - "" + "", ], ), ( @@ -53,7 +53,7 @@ class TestListOutputs: " }", " ]", "}", - "" + "", ], ), ], @@ -89,7 +89,7 @@ def test_list_outputs( status_result_per_line = status_result.stdout.split("\n") workflow_outputs = [ - sub.replace('', test_workflow_id) for sub in output_template + sub.replace("", test_workflow_id) for sub in output_template ] print("Print workflow list-outputs results:") From ce7069737b8cd6ab340ad5d38019882806ec2814 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 10:57:47 -0400 Subject: [PATCH 14/20] fixed function name spelling --- src/cromshell/list_outputs/command.py | 8 +++++--- tests/unit/test_list_outputs.py | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index 4bc79d7e..f5c657d0 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -106,10 +106,10 @@ def get_task_level_outputs(config) -> dict: headers=http_utils.generate_headers(config), ) - return filer_outputs_from_workflow_metadata(workflow_metadata) + return filter_outputs_from_workflow_metadata(workflow_metadata) -def filer_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: +def filter_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: """Get the outputs from the workflow metadata Args: @@ -124,7 +124,9 @@ def filer_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: output_metadata[call] = [] for scatter in calls_metadata[call]: output_metadata[call].append( - filer_outputs_from_workflow_metadata(scatter["subWorkflowMetadata"]) + filter_outputs_from_workflow_metadata( + scatter["subWorkflowMetadata"] + ) ) else: output_metadata[call] = [] diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index c7dca44b..a487d443 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -8,9 +8,6 @@ class TestListOutputs: """Test the execution list-outputs command functions""" - # def get_workflow_level_outputs(config) -> dict: - # def test_get_task_level_outputs(config) -> dict: - @pytest.mark.parametrize( "workflow_metadata_file, outputs_metadata_file_path", [ @@ -34,7 +31,9 @@ def test_filer_outputs_from_workflow_metadata( outputs_metadata = json.load(f) assert ( - list_outputs_command.filer_outputs_from_workflow_metadata(workflow_metadata) + list_outputs_command.filter_outputs_from_workflow_metadata( + workflow_metadata + ) == outputs_metadata ) From 0071d01a52866a1e7344583bcfa4bb26dbe13d2d Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 10:59:08 -0400 Subject: [PATCH 15/20] alternative to adding options to cromshell in integration tests --- tests/integration/test_list_outputs.py | 2 +- tests/integration/utility_test_functions.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_list_outputs.py b/tests/integration/test_list_outputs.py index 754ae545..27ab8351 100644 --- a/tests/integration/test_list_outputs.py +++ b/tests/integration/test_list_outputs.py @@ -83,7 +83,7 @@ def test_list_outputs( status_result = utility_test_functions.run_cromshell_command( command=["list-outputs", test_workflow_id], exit_code=0, - options=options, + subcommand_options=options, ) status_result_per_line = status_result.stdout.split("\n") diff --git a/tests/integration/utility_test_functions.py b/tests/integration/utility_test_functions.py index e7d8034d..c628f0fe 100644 --- a/tests/integration/utility_test_functions.py +++ b/tests/integration/utility_test_functions.py @@ -10,11 +10,13 @@ from cromshell.utilities import cromshellconfig -def run_cromshell_command(command: list, exit_code: int, options: list = None): +def run_cromshell_command( + command: list, exit_code: int, subcommand_options: list = None +): """ Run cromshell alias using CliRunner and assert job is successful - :param options: + :param subcommand_options: The options to pass to the subcommand :param command: The subcommand, options, and arguments in list form e.g. [ "alias", @@ -26,10 +28,10 @@ def run_cromshell_command(command: list, exit_code: int, options: list = None): :return: results from execution """ - command_with_options = command.copy() - if options: - for item in options: - command_with_options.insert(1, item) + if subcommand_options: + command_with_options = command[:1] + subcommand_options + command[1:] + else: + command_with_options = command runner = CliRunner(mix_stderr=False) # The absolute path will be passed to the invoke command because From 7b3b20e7235d08799de26d36989e814b306a41a0 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 14:24:08 -0400 Subject: [PATCH 16/20] Added function to confirm results from cromwell outputs endpoint contain outputs else throws error. Added variable to hold workflow id in cromshellconfig.py --- src/cromshell/list_outputs/command.py | 16 +++++++++- .../utilities/command_setup_utils.py | 12 +++++++ src/cromshell/utilities/cromshellconfig.py | 1 + tests/unit/test_list_outputs.py | 31 +++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index f5c657d0..0966b679 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -74,10 +74,12 @@ def get_workflow_level_outputs(config) -> dict: ) if requests_out.ok: + check_for_empty_output(requests_out.json(), config.workflow_id) return requests_out.json() else: http_utils.check_http_request_status_code( - short_error_message="Failed to retrieve workflow outputs.", + short_error_message="Failed to retrieve outputs for " + f"workflow: {config.workflow_id}", response=requests_out, # Raising exception is set false to allow # command to retrieve outputs of remaining workflows. @@ -200,3 +202,15 @@ def is_path_or_url_like(in_string: str) -> bool: return True else: return False + + +def check_for_empty_output(cromwell_outputs: dict, workflow_id: str) -> None: + """Check if the workflow outputs are empty + + Args: + cromwell_outputs (dict): Results from cromwell server output endpoint + :param workflow_id: The workflow id + """ + if not cromwell_outputs.get("outputs"): + LOGGER.error(f"No outputs found for workflow: {workflow_id}") + raise Exception(f"No outputs found for workflow: {workflow_id}") diff --git a/src/cromshell/utilities/command_setup_utils.py b/src/cromshell/utilities/command_setup_utils.py index 48221674..12bc90b2 100644 --- a/src/cromshell/utilities/command_setup_utils.py +++ b/src/cromshell/utilities/command_setup_utils.py @@ -17,5 +17,17 @@ def resolve_workflow_id_and_server(workflow_id: str, cromshell_config) -> str: http_utils.set_and_check_cromwell_server( config=cromshell_config, workflow_id=resolved_workflow_id ) + set_workflow_id(workflow_id=resolved_workflow_id, cromshell_config=cromshell_config) return resolved_workflow_id + + +def set_workflow_id(workflow_id: str, cromshell_config) -> None: + """ + Sets the workflow id in the config object + + :param workflow_id: workflow UUID + :param cromshell_config: + :return: None + """ + cromshell_config.workflow_id = workflow_id diff --git a/src/cromshell/utilities/cromshellconfig.py b/src/cromshell/utilities/cromshellconfig.py index a5b37108..163f884f 100644 --- a/src/cromshell/utilities/cromshellconfig.py +++ b/src/cromshell/utilities/cromshellconfig.py @@ -27,6 +27,7 @@ ] CROMWELL_API_STRING = "/api/workflows/v1" WOMTOOL_API_STRING = "/api/womtool/v1" +workflow_id = None # Concatenate the cromwell url, api string, and workflow ID. Set in subcommand. cromwell_api_workflow_id = None # Defaults for variables will be set after functions have been defined diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index a487d443..ebbc9102 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -172,3 +172,34 @@ def test_print_output_name_and_file( ) def test_is_path_or_url_like(self, value, expected_bool): assert list_outputs_command.is_path_or_url_like(value) == expected_bool + + + @pytest.mark.parametrize( + "example_output_results, workflow_id", + [ + [ + {'outputs': {}, 'id': '04b65be4-896f-439c-8a01-5e4dc6c116dd'}, + "04b65be4-896f-439c-8a01-5e4dc6c116dd'", + ], + [ + {'outputs': {"one": 2}, 'id': '04b65be4-896f-439c-8a01-5e4dc6c116dd'}, + "04b65be4-896f-439c-8a01-5e4dc6c116dd'", + ], + ], + ) + def test_check_for_empty_output( + self, example_output_results: dict, workflow_id: str + ): + """Test the check_for_empty_output function""" + + if example_output_results.get("outputs") == {}: + with pytest.raises(Exception): + list_outputs_command.check_for_empty_output( + example_output_results, workflow_id + ) + else: + assert ( + list_outputs_command.check_for_empty_output( + example_output_results, workflow_id + ) is None + ) \ No newline at end of file From 200be24c4049b2b64dbe16b1851ce59daebb29a9 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 14:59:41 -0400 Subject: [PATCH 17/20] add check of outputs for detailed list-outputs option --- src/cromshell/list_outputs/command.py | 10 ++++++---- tests/unit/test_list_outputs.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index 0966b679..c03264ba 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -74,7 +74,7 @@ def get_workflow_level_outputs(config) -> dict: ) if requests_out.ok: - check_for_empty_output(requests_out.json(), config.workflow_id) + check_for_empty_output(requests_out.json().get("outputs"), config.workflow_id) return requests_out.json() else: http_utils.check_http_request_status_code( @@ -135,6 +135,8 @@ def filter_outputs_from_workflow_metadata(workflow_metadata: dict) -> dict: for index in index_list: output_metadata[call].append(index.get(extract_task_key)) + check_for_empty_output(output_metadata, workflow_metadata["id"]) + return output_metadata @@ -204,13 +206,13 @@ def is_path_or_url_like(in_string: str) -> bool: return False -def check_for_empty_output(cromwell_outputs: dict, workflow_id: str) -> None: +def check_for_empty_output(workflow_outputs: dict, workflow_id: str) -> None: """Check if the workflow outputs are empty Args: - cromwell_outputs (dict): Results from cromwell server output endpoint + cromwell_outputs (dict): Dictionary of workflow outputs :param workflow_id: The workflow id """ - if not cromwell_outputs.get("outputs"): + if not workflow_outputs: LOGGER.error(f"No outputs found for workflow: {workflow_id}") raise Exception(f"No outputs found for workflow: {workflow_id}") diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index ebbc9102..acfcc279 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -178,7 +178,7 @@ def test_is_path_or_url_like(self, value, expected_bool): "example_output_results, workflow_id", [ [ - {'outputs': {}, 'id': '04b65be4-896f-439c-8a01-5e4dc6c116dd'}, + {}, "04b65be4-896f-439c-8a01-5e4dc6c116dd'", ], [ @@ -192,7 +192,7 @@ def test_check_for_empty_output( ): """Test the check_for_empty_output function""" - if example_output_results.get("outputs") == {}: + if example_output_results == {}: with pytest.raises(Exception): list_outputs_command.check_for_empty_output( example_output_results, workflow_id @@ -202,4 +202,4 @@ def test_check_for_empty_output( list_outputs_command.check_for_empty_output( example_output_results, workflow_id ) is None - ) \ No newline at end of file + ) From 388d34e8ebb385cd6aef504dbea6abc1986ab0b9 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 15:16:14 -0400 Subject: [PATCH 18/20] add check of outputs for detailed list-outputs option --- src/cromshell/list_outputs/command.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index c03264ba..bad23dd5 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -41,20 +41,22 @@ def main(config, workflow_ids, detailed, json_summary): ) if not detailed: + workflow_outputs = get_workflow_level_outputs(config).get("outputs") + if json_summary: - io_utils.pretty_print_json( - format_json=get_workflow_level_outputs(config).get("outputs") - ) + io_utils.pretty_print_json(format_json=workflow_outputs) else: print_file_like_value_in_dict( - outputs_metadata=get_workflow_level_outputs(config).get("outputs"), + outputs_metadata=workflow_outputs, indent=False, ) else: + task_outputs = get_task_level_outputs(config) + if json_summary: - io_utils.pretty_print_json(format_json=get_task_level_outputs(config)) + io_utils.pretty_print_json(format_json=task_outputs) else: - print_task_level_outputs(get_task_level_outputs(config)) + print_task_level_outputs(task_outputs) return return_code From c3b87a7664c5715b0c0cb967196b4dffac6ee9cf Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 15:23:44 -0400 Subject: [PATCH 19/20] lint fixes --- src/cromshell/list_outputs/command.py | 2 +- tests/unit/test_list_outputs.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cromshell/list_outputs/command.py b/src/cromshell/list_outputs/command.py index bad23dd5..06a23a4a 100644 --- a/src/cromshell/list_outputs/command.py +++ b/src/cromshell/list_outputs/command.py @@ -81,7 +81,7 @@ def get_workflow_level_outputs(config) -> dict: else: http_utils.check_http_request_status_code( short_error_message="Failed to retrieve outputs for " - f"workflow: {config.workflow_id}", + f"workflow: {config.workflow_id}", response=requests_out, # Raising exception is set false to allow # command to retrieve outputs of remaining workflows. diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index acfcc279..3bc6ee11 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -173,7 +173,6 @@ def test_print_output_name_and_file( def test_is_path_or_url_like(self, value, expected_bool): assert list_outputs_command.is_path_or_url_like(value) == expected_bool - @pytest.mark.parametrize( "example_output_results, workflow_id", [ @@ -182,7 +181,7 @@ def test_is_path_or_url_like(self, value, expected_bool): "04b65be4-896f-439c-8a01-5e4dc6c116dd'", ], [ - {'outputs': {"one": 2}, 'id': '04b65be4-896f-439c-8a01-5e4dc6c116dd'}, + {"outputs": {"one": 2}, "id": "04b65be4-896f-439c-8a01-5e4dc6c116dd"}, "04b65be4-896f-439c-8a01-5e4dc6c116dd'", ], ], @@ -201,5 +200,6 @@ def test_check_for_empty_output( assert ( list_outputs_command.check_for_empty_output( example_output_results, workflow_id - ) is None + ) + is None ) From 5b11e497cba6bb364aac0bd8fe2e8ef35ce5fbde Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 31 Mar 2023 15:25:16 -0400 Subject: [PATCH 20/20] spelling --- tests/unit/test_list_outputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_list_outputs.py b/tests/unit/test_list_outputs.py index 3bc6ee11..e84856d3 100644 --- a/tests/unit/test_list_outputs.py +++ b/tests/unit/test_list_outputs.py @@ -17,7 +17,7 @@ class TestListOutputs: ], ], ) - def test_filer_outputs_from_workflow_metadata( + def test_filter_outputs_from_workflow_metadata( self, mock_data_path, tests_metadata_path,