Skip to content

Commit

Permalink
Centralize logging memory usage
Browse files Browse the repository at this point in the history
The same snippet responsible for logging memory usage was repeated
multiple times in ``drivers.py`` module.  Factord it out as an auxiliary
function to avoid repetitions.
  • Loading branch information
mxk62 committed Dec 4, 2024
1 parent 5882ad3 commit 7fb8495
Showing 1 changed file with 21 additions and 47 deletions.
68 changes: 21 additions & 47 deletions python/lsst/ctrl/bps/drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,7 @@ def _init_submission_driver(config_file: str, **kwargs) -> BpsConfig:
mem_fmt=DEFAULT_MEM_FMT,
):
config = init_submission(config_file, validators=validators, **kwargs)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 106 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L106

Added line #L106 was not covered by tests

submit_path = config[".bps_defined.submitPath"]

Check warning on line 108 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L108

Added line #L108 was not covered by tests
print(f"Submit dir: {submit_path}")
Expand Down Expand Up @@ -145,11 +141,7 @@ def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, Quantu
mem_fmt=DEFAULT_MEM_FMT,
):
qgraph_file, qgraph = acquire_quantum_graph(config, out_prefix=submit_path)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 144 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L144

Added line #L144 was not covered by tests

config[".bps_defined.runQgraphFile"] = qgraph_file
return config, qgraph
Expand Down Expand Up @@ -185,11 +177,8 @@ def cluster_qgraph_driver(config_file, **kwargs):
mem_fmt=DEFAULT_MEM_FMT,
):
clustered_qgraph = cluster_quanta(config, qgraph, config["uniqProcName"])
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 180 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L180

Added line #L180 was not covered by tests

_LOG.info("ClusteredQuantumGraph contains %d cluster(s)", len(clustered_qgraph))

submit_path = config[".bps_defined.submitPath"]
Expand Down Expand Up @@ -235,11 +224,8 @@ def transform_driver(config_file, **kwargs):
):
generic_workflow, generic_workflow_config = transform(config, clustered_qgraph, submit_path)
_LOG.info("Generic workflow name '%s'", generic_workflow.name)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 227 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L227

Added line #L227 was not covered by tests

num_jobs = sum(generic_workflow.job_counts.values())
_LOG.info("GenericWorkflow contains %d job(s) (including final)", num_jobs)

Expand Down Expand Up @@ -287,11 +273,7 @@ def prepare_driver(config_file, **kwargs):
mem_fmt=DEFAULT_MEM_FMT,
):
wms_workflow = prepare(generic_workflow_config, generic_workflow, submit_path)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 276 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L276

Added line #L276 was not covered by tests

wms_workflow_config = generic_workflow_config
return wms_workflow_config, wms_workflow
Expand Down Expand Up @@ -377,11 +359,7 @@ def submit_driver(config_file, **kwargs):
if not wms_workflow:
wms_workflow = workflow
_LOG.info("Run '%s' submitted for execution with id '%s'", wms_workflow.name, wms_workflow.run_id)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 362 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L362

Added line #L362 was not covered by tests

_make_id_link(wms_workflow_config, wms_workflow.run_id)

Expand Down Expand Up @@ -583,11 +561,8 @@ def submitcmd_driver(config_file: str, **kwargs) -> None:
mem_fmt=DEFAULT_MEM_FMT,
):
config = init_submission(config_file, validators=validators, **kwargs)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 564 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L563-L564

Added lines #L563 - L564 were not covered by tests

submit_path = config[".bps_defined.submitPath"]

Check warning on line 566 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L566

Added line #L566 was not covered by tests

_LOG.info("Starting construction stage (creating generic workflow)")
Expand All @@ -602,11 +577,8 @@ def submitcmd_driver(config_file: str, **kwargs) -> None:
):
generic_workflow, generic_workflow_config = construct(config)
_LOG.info("Generic workflow name '%s'", generic_workflow.name)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 580 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L578-L580

Added lines #L578 - L580 were not covered by tests

_, save_workflow = config.search("saveGenericWorkflow", opt={"default": False})

Check warning on line 582 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L582

Added line #L582 was not covered by tests
if save_workflow:
with open(os.path.join(submit_path, "bps_generic_workflow.pickle"), "wb") as outfh:
Expand All @@ -627,11 +599,8 @@ def submitcmd_driver(config_file: str, **kwargs) -> None:
mem_fmt=DEFAULT_MEM_FMT,
):
wms_workflow = prepare(generic_workflow_config, generic_workflow, submit_path)
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
_log_mem_usage()

Check warning on line 602 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L601-L602

Added lines #L601 - L602 were not covered by tests

wms_workflow_config = generic_workflow_config

Check warning on line 604 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L604

Added line #L604 was not covered by tests

if kwargs.get("dry_run", False):
Expand All @@ -648,10 +617,15 @@ def submitcmd_driver(config_file: str, **kwargs) -> None:
mem_fmt=DEFAULT_MEM_FMT,
):
submit(wms_workflow_config, wms_workflow, **kwargs)
_log_mem_usage()
print(f"Run Id: {wms_workflow.run_id}")
print(f"Run Name: {wms_workflow.name}")

Check warning on line 622 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L619-L622

Added lines #L619 - L622 were not covered by tests


def _log_mem_usage() -> None:
"""Log memory usage."""
if _LOG.isEnabledFor(logging.INFO):
_LOG.info(

Check warning on line 628 in python/lsst/ctrl/bps/drivers.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/ctrl/bps/drivers.py#L628

Added line #L628 was not covered by tests
"Peak memory usage for bps process %s (main), %s (largest child process)",
*tuple(f"{val.to(DEFAULT_MEM_UNIT):{DEFAULT_MEM_FMT}}" for val in get_peak_mem_usage()),
)
print(f"Run Id: {wms_workflow.run_id}")
print(f"Run Name: {wms_workflow.name}")

0 comments on commit 7fb8495

Please sign in to comment.