From 99b96f83d1d1698dd281e57e043a0cb917efea66 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 26 Jul 2024 07:27:30 +0200 Subject: [PATCH] Show progress bar The bootstrap, build-sequence and download-sequence commands now use `tqdm` to show a progress bar. It's now easier to see how far along a long-running pipeline job has come. Signed-off-by: Christian Heimes --- requirements.txt | 4 +++- src/fromager/commands/bootstrap.py | 4 ++-- src/fromager/commands/build.py | 11 ++++++----- src/fromager/commands/download_sequence.py | 4 ++-- src/fromager/progress.py | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 src/fromager/progress.py diff --git a/requirements.txt b/requirements.txt index 0ddd569f..35a2bf87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ requests resolvelib stevedore toml -virtualenv \ No newline at end of file +tqdm +virtualenv + diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index 4ea3f92d..5684a7a6 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -5,7 +5,7 @@ import click from packaging.requirements import Requirement -from .. import clickext, context, requirements_file, sdist, server +from .. import clickext, context, progress, requirements_file, sdist, server logger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def bootstrap( server.start_wheel_server(wkctx) - for origin, toplevel in to_build: + for origin, toplevel in progress.progress(to_build): sdist.handle_requirement(wkctx, Requirement(toplevel), req_type=origin) # If we put pre-built wheels in the downloads directory, we should diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index a3b80346..e43b1ae5 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -6,7 +6,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from fromager import clickext, context, hooks, sdist, server, sources, wheels +from fromager import clickext, context, hooks, progress, sdist, server, sources, wheels logger = logging.getLogger(__name__) @@ -67,10 +67,11 @@ def build_sequence( """ with open(build_order_file, "r") as f: - for entry in json.load(f): - wheel_filename = _build( - wkctx, entry["dist"], Version(entry["version"]), sdist_server_url - ) + for entry in progress.progress(json.load(f)): + dist_name = entry["dist"] + dist_version = Version(entry["version"]) + logger.info("%s: Building %s==%s", dist_name, dist_name, dist_version) + wheel_filename = _build(wkctx, dist_name, dist_version, sdist_server_url) server.update_wheel_mirror(wkctx) # After we update the wheel mirror, the built file has # moved to a new directory. diff --git a/src/fromager/commands/download_sequence.py b/src/fromager/commands/download_sequence.py index 9e904f80..8ec070e4 100644 --- a/src/fromager/commands/download_sequence.py +++ b/src/fromager/commands/download_sequence.py @@ -4,7 +4,7 @@ import click from packaging.requirements import Requirement -from .. import context, sdist, sources +from .. import context, progress, sdist, sources logger = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def download_sequence( wheel_servers = [sdist_server_url] with open(build_order_file, "r") as f: - for entry in json.load(f): + for entry in progress.progress(json.load(f)): if entry["prebuilt"]: logger.info(f"{entry['dist']} uses a pre-built wheel, skipping") continue diff --git a/src/fromager/progress.py b/src/fromager/progress.py new file mode 100644 index 00000000..34cfc75f --- /dev/null +++ b/src/fromager/progress.py @@ -0,0 +1,14 @@ +import sys +import typing + +import tqdm as _tqdm + +__all__ = ("progress",) + + +def progress(it: typing.Iterable, *, unit="pkg", **kwargs: typing.Any) -> typing.Any: + """tqdm progress bar""" + if not sys.stdout.isatty(): + # wider progress bar in CI + kwargs.setdefault("ncols", 78) + yield from _tqdm.tqdm(it, unit=unit, **kwargs)