From 56b467badf5cf61a2f42ada538b41038efa208d1 Mon Sep 17 00:00:00 2001 From: Graeme Winter Date: Mon, 25 Nov 2024 10:32:34 +0000 Subject: [PATCH 1/4] Add tqdm progress bar to dials.import (#768) Give the user some hint as to the progress i.e. the computer is doing something when you import thousands of files. Only show progress bar if > 1 file and isatty, includes a cast of the `filenames` iterator to a list to allow evaluation of the length for progress assessment. Resolves dials/dials#2783 --- newsfragments/768.feature | 2 ++ src/dxtbx/model/experiment_list.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 newsfragments/768.feature diff --git a/newsfragments/768.feature b/newsfragments/768.feature new file mode 100644 index 000000000..9ac92167c --- /dev/null +++ b/newsfragments/768.feature @@ -0,0 +1,2 @@ +``dials.import``: add a progress bar for the import process, particularly +useful when importing thousands of files diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index 1386fd231..6f3590c5f 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -14,6 +14,7 @@ from typing import Any, Callable, Generator, Iterable import natsort +from tqdm import tqdm import dxtbx from dxtbx.format.Format import Format @@ -664,6 +665,9 @@ def from_filenames( """Create a list of data blocks from a list of directory or file names.""" experiments = ExperimentList() + # Cast filenames to a list from whatever iterator they are + filenames = list(filenames) + # Process each file given by this path list to_process = _openingpathiterator(filenames) find_format = FormatChecker() @@ -671,7 +675,13 @@ def from_filenames( format_groups = collections.defaultdict(list) if format_kwargs is None: format_kwargs = {} - for filename in to_process: + + if os.isatty and len(filenames) > 1: + filename_iter = tqdm(to_process, total=len(filenames)) + else: + filename_iter = to_process + + for filename in filename_iter: # We now have a file, pre-opened by Format.open_file (therefore # cached). Determine its type, and prepare to put into a group format_class = find_format.find_format(filename) From 8d36ec9b80c5a4dfb6c81743f3e7ab73780b12f1 Mon Sep 17 00:00:00 2001 From: Graeme Winter Date: Mon, 25 Nov 2024 16:17:08 +0000 Subject: [PATCH 2/4] Bugfix: stdout output for tqdm (#773) Bugfix: stdout output for tqdm --- newsfragments/773.bugfix | 1 + src/dxtbx/model/experiment_list.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/773.bugfix diff --git a/newsfragments/773.bugfix b/newsfragments/773.bugfix new file mode 100644 index 000000000..a9df61b72 --- /dev/null +++ b/newsfragments/773.bugfix @@ -0,0 +1 @@ +``dials.import``: force tqdm output to stdout not stderr as default \ No newline at end of file diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index 6f3590c5f..a2f62f4b1 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -677,7 +677,7 @@ def from_filenames( format_kwargs = {} if os.isatty and len(filenames) > 1: - filename_iter = tqdm(to_process, total=len(filenames)) + filename_iter = tqdm(to_process, total=len(filenames), file=sys.stdout) else: filename_iter = to_process From c46a75848bb087e6b9f55331527bc04280701029 Mon Sep 17 00:00:00 2001 From: Graeme Winter Date: Mon, 25 Nov 2024 17:05:30 +0000 Subject: [PATCH 3/4] Hide tqdm if DIALS_NOBANNER (#774) --- newsfragments/774.bugfix | 1 + src/dxtbx/model/experiment_list.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 newsfragments/774.bugfix diff --git a/newsfragments/774.bugfix b/newsfragments/774.bugfix new file mode 100644 index 000000000..6f355b5f5 --- /dev/null +++ b/newsfragments/774.bugfix @@ -0,0 +1 @@ +``dials.show``: hide tqdm if DIALS_NOBANNER diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index a2f62f4b1..9916636f5 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -676,8 +676,8 @@ def from_filenames( if format_kwargs is None: format_kwargs = {} - if os.isatty and len(filenames) > 1: - filename_iter = tqdm(to_process, total=len(filenames), file=sys.stdout) + if os.isatty and len(filenames) > 1 and "DIALS_NOBANNER" not in os.environ: + filename_iter = tqdm(to_process, total=len(filenames)) else: filename_iter = to_process From edb60e4b842036fd4678feb2f84d41627fd84b59 Mon Sep 17 00:00:00 2001 From: Graeme Winter Date: Tue, 26 Nov 2024 08:31:39 +0000 Subject: [PATCH 4/4] TQDM -> stdout (#775) * Reinstate tqdm -> stdout --- newsfragments/775.bugfix | 1 + src/dxtbx/model/experiment_list.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/775.bugfix diff --git a/newsfragments/775.bugfix b/newsfragments/775.bugfix new file mode 100644 index 000000000..450b4bd4a --- /dev/null +++ b/newsfragments/775.bugfix @@ -0,0 +1 @@ +``dials.import``: set tqdm output to stdout (again) diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index 9916636f5..4ab93b966 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -677,7 +677,7 @@ def from_filenames( format_kwargs = {} if os.isatty and len(filenames) > 1 and "DIALS_NOBANNER" not in os.environ: - filename_iter = tqdm(to_process, total=len(filenames)) + filename_iter = tqdm(to_process, total=len(filenames), file=sys.stdout) else: filename_iter = to_process