fix broken pbar with tqdm pbar in all modules

drtobybrown · Oct 21, 2024 · 06626b2 · 06626b2
1 parent aa3fd7a
commit 06626b2
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 43 deletions.
diff --git a/.github/workflows/test_build_mac.yaml b/.github/workflows/test_build_mac.yaml
@@ -23,7 +23,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy astropy PyQt6 scipy matplotlib pytest latex spectral_cube extinction h5py joblib
+          pip install numpy astropy PyQt6 scipy matplotlib pytest spectral_cube extinction h5py joblib tqdm
           # force reinstall of numpy due to v2.0.0 not working properly with other packages
           pip install --force-reinstall -v "numpy==1.26.4"          
 

diff --git a/.github/workflows/test_build_ubuntu.yaml b/.github/workflows/test_build_ubuntu.yaml
@@ -26,7 +26,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy astropy PyQt6 scipy matplotlib pytest latex spectral_cube extinction h5py joblib
+          pip install numpy astropy PyQt6 scipy matplotlib pytest spectral_cube extinction h5py joblib tqdm
           # force reinstall of numpy due to v2.0.0 not working properly with other packages
           pip install --force-reinstall -v "numpy==1.26.4"
           

diff --git a/gistPipeline/continuumCube/ppxf_cont_wrapper.py b/gistPipeline/continuumCube/ppxf_cont_wrapper.py
@@ -12,6 +12,7 @@
 from joblib import Parallel, delayed, dump, load
 from ppxf.ppxf import ppxf
 from printStatus import printStatus
+from tqdm import tqdm
 
 # PHYSICAL CONSTANTS
 C = 299792.458  # km/s
@@ -73,7 +74,7 @@ def run_ppxf(
     ui.adsabs.harvard.edu/?#abs/2017MNRAS.466..798C), in order to determine the
     stellar kinematics.
     """
-    printStatus.progressBar(i, nbins, barLength=50)
+    # printStatus.progressBar(i, nbins, barLength=50)
 
     try:
         # normalise galaxy spectra and noise
@@ -530,10 +531,11 @@ def worker(chunk, templates):
 
         # Use joblib to parallelize the work
         max_nbytes = "1M" # max array size before memory mapping is triggered
-        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"]))
+        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"] * 10))
         chunks = [range(i, min(i + chunk_size, nbins)) for i in range(0, nbins, chunk_size)]
-        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c"}
-        ppxf_tmp = Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks)
+        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c", "return_as":"generator"}
+        ppxf_tmp = list(tqdm(Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks),
+                        total=len(chunks), desc="Processing Chunks"))
 
         # Flatten the results
         ppxf_tmp = [result for chunk_results in ppxf_tmp for result in chunk_results]
@@ -547,7 +549,7 @@ def worker(chunk, templates):
             formal_error[i, : config["CONT"]["MOM"]] = ppxf_tmp[i][5]
             spectral_mask[i, :] = ppxf_tmp[i][6]
 
-        printStatus.updateDone("Running PPXF in parallel mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in parallel mode", progressbar=False)
 
     elif config["GENERAL"]["PARALLEL"] == False:
         printStatus.running("Running PPXF in serial mode")
@@ -581,7 +583,7 @@ def worker(chunk, templates):
                 i,
                 optimal_template_comb,
             )
-        printStatus.updateDone("Running PPXF in serial mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in serial mode", progressbar=False)
 
     print(
         "             Running PPXF on %s spectra took %.2fs using %i cores"

diff --git a/gistPipeline/emissionLines/ppxf_gas_wrapper.py b/gistPipeline/emissionLines/ppxf_gas_wrapper.py
@@ -1,4 +1,3 @@
-import glob
 import logging
 import os
 import time
@@ -14,6 +13,7 @@
 # Then use system installed version instead
 from ppxf.ppxf import ppxf
 from printStatus import printStatus
+from tqdm import tqdm
 
 # Physical constants
 C = 299792.458  # speed of light in km/s
@@ -53,7 +53,7 @@ def run_ppxf(
     ui.adsabs.harvard.edu/?#abs/2017MNRAS.466..798C), in order to determine the
     non-parametric star-formation histories.
     """
-    printStatus.progressBar(i, np.max(ubins) + 1, barLength=50)
+    # printStatus.progressBar(i, np.max(ubins) + 1, barLength=50)
 
     try:
         pp = ppxf(
@@ -905,10 +905,11 @@ def worker(chunk, templates):
 
         # Use joblib to parallelize the work
         max_nbytes = "1M" # max array size before memory mapping is triggered
-        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"]))
+        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"] * 10))
         chunks = [range(i, min(i + chunk_size, nbins)) for i in range(0, nbins, chunk_size)]
-        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c"}
-        ppxf_tmp = Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks)
+        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c", "return_as":"generator"}
+        ppxf_tmp = list(tqdm(Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks),
+                        total=len(chunks), desc="Processing Chunks"))
 
         # Flatten the results
         ppxf_tmp = [result for chunk_results in ppxf_tmp for result in chunk_results]
@@ -924,7 +925,7 @@ def worker(chunk, templates):
             stkin[i, :] = ppxf_tmp[i][8]
             stkin_err[i, :] = ppxf_tmp[i][9]
 
-        printStatus.updateDone("Running PPXF in parallel mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in parallel mode", progressbar=False)
 
     elif config["GENERAL"]["PARALLEL"] == False:
         printStatus.running("Running PPXF in serial mode")
@@ -965,7 +966,7 @@ def worker(chunk, templates):
                 ubins,
             )
 
-        printStatus.updateDone("Running PPXF in serial mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in serial mode", progressbar=False)
 
     print(
         "             Running PPXF on %s spectra took %.2fs"

diff --git a/gistPipeline/lineStrengths/default.py b/gistPipeline/lineStrengths/default.py
@@ -11,6 +11,7 @@
 from joblib import Parallel, delayed, dump, load
 from ppxf.ppxf_util import gaussian_filter1d
 from printStatus import printStatus
+from tqdm import tqdm
 
 cvel = 299792.458
 
@@ -72,7 +73,7 @@ def run_ls(
     tuple: A tuple of indices, errors, vals, and percentiles
     """
     # Display progress bar
-    printStatus.progressBar(i, nbins, barLength=50)
+    # printStatus.progressBar(i, nbins, barLength=50)
     nindex = len(index_names)
 
     try:
@@ -401,20 +402,20 @@ def measureLineStrengths(config, RESOLUTION="ORIGINAL"):
         # Rebin the cleaned spectra from log to lin
         printStatus.running("Rebinning the spectra from log to lin")
         for i in range(nbins):
-            printStatus.progressBar(i, nbins, barLength=50)
+            # printStatus.progressBar(i, nbins, barLength=50)
             spec[i, :], wave = log_unbinning(lamRange, oldspec[i, :])
         printStatus.updateDone(
-            "Rebinning the spectra from log to lin", progressbar=True
+            "Rebinning the spectra from log to lin", progressbar=False
         )
 
         # Rebin the error spectra from log to lin
         printStatus.running("Rebinning the error spectra from log to lin")
 
         for i in range(nbins):
-            printStatus.progressBar(i, nbins, barLength=50)
+            # printStatus.progressBar(i, nbins, barLength=50)
             espec[i, :], _ = log_unbinning(lamRange, oldespec[i, :])
         printStatus.updateDone(
-            "Rebinning the error spectra from log to lin", progressbar=True
+            "Rebinning the error spectra from log to lin", progressbar=False
         )
 
         # Save cleaned, linear spectra
@@ -463,7 +464,7 @@ def measureLineStrengths(config, RESOLUTION="ORIGINAL"):
             velscale = f.attrs['VELSCALE']
         # Iterate over all bins
         for i in range(0, nbins):
-            printStatus.progressBar(i, nbins, barLength=50)
+            # printStatus.progressBar(i, nbins, barLength=50)
 
             # Convert velocity dispersion of galaxy (from PPXF) to Angstrom
             veldisp_kin_Angst = veldisp_kin[i] * wave / cvel * 2.355
@@ -487,7 +488,7 @@ def measureLineStrengths(config, RESOLUTION="ORIGINAL"):
             spec[i, :] = gaussian_filter1d(spec[i, :], sigma)
             espec[i, :] = gaussian_filter1d(espec[i, :], sigma)
         printStatus.updateDone(
-            "Broadening the spectra to LIS resolution", progressbar=True
+            "Broadening the spectra to LIS resolution", progressbar=False
         )
 
     # Get indices that are considered in SSP-conversion
@@ -555,10 +556,11 @@ def worker(chunk):
 
         # Use joblib to parallelize the work
         max_nbytes = None  # max array size before memory mapping is triggered (None = disabled memory mapping, see https://github.com/scikit-learn-contrib/hdbscan/pull/495#issue-1014324032)
-        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"]))
+        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"] * 10))
         chunks = [range(i, min(i + chunk_size, nbins)) for i in range(0, nbins, chunk_size)]
-        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "mmap_mode": "c", "temp_folder":memmap_folder}
-        ppxf_tmp = Parallel(**parallel_configs)(delayed(worker)(chunk) for chunk in chunks)
+        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c", "return_as":"generator"}
+        ppxf_tmp = list(tqdm(Parallel(**parallel_configs)(delayed(worker)(chunk) for chunk in chunks),
+                        total=len(chunks), desc="Processing Chunks"))
 
         # Flatten the results
         ppxf_tmp = [result for chunk_results in ppxf_tmp for result in chunk_results]
@@ -569,7 +571,7 @@ def worker(chunk):
                 vals[i, :], percentile[i, :, :] = extra
 
         printStatus.updateDone(
-            "Running lineStrengths in parallel mode", progressbar=True
+            "Running lineStrengths in parallel mode", progressbar=False
         )
 
     if config["GENERAL"]["PARALLEL"] == False:
@@ -620,7 +622,7 @@ def worker(chunk):
                     MCMC,
                 )
 
-        printStatus.updateDone("Running lineStrengths in serial mode", progressbar=True)
+        printStatus.updateDone("Running lineStrengths in serial mode", progressbar=False)
 
     print(
         "             Running lineStrengths on %s spectra took %.2fs using %i cores"

diff --git a/gistPipeline/starFormationHistories/ppxf_sfh_wrapper.py b/gistPipeline/starFormationHistories/ppxf_sfh_wrapper.py
@@ -14,6 +14,7 @@
 from packaging import version
 from ppxf.ppxf import ppxf
 from printStatus import printStatus
+from tqdm import tqdm
 
 # Physical constants
 C = 299792.458  # speed of light in km/s
@@ -89,7 +90,7 @@ def run_ppxf_firsttime(
         start,
         goodpixels=goodPixels,
         plot=False,
-        quiet=False,
+        quiet=True,
         moments=nmoments,
         degree=-1,
         vsyst=offset,
@@ -142,7 +143,7 @@ def run_ppxf(
     ui.adsabs.harvard.edu/?#abs/2017MNRAS.466..798C), in order to determine the
     non-parametric star-formation histories.
     """
-    printStatus.progressBar(i, nbins, barLength=50)
+    # printStatus.progressBar(i, nbins, barLength=50)
 
     try:
 
@@ -890,10 +891,11 @@ def worker(chunk, templates):
 
         # Use joblib to parallelize the work
         max_nbytes = "1M" # max array size before memory mapping is triggered
-        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"]))
+        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"] * 10))
         chunks = [range(i, min(i + chunk_size, nbins)) for i in range(0, nbins, chunk_size)]
-        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c"}
-        ppxf_tmp = Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks)
+        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c", "return_as":"generator"}
+        ppxf_tmp = list(tqdm(Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks),
+                        total=len(chunks), desc="Processing Chunks"))
 
         # Flatten the results
         ppxf_tmp = [result for chunk_results in ppxf_tmp for result in chunk_results]
@@ -915,7 +917,7 @@ def worker(chunk, templates):
             snr_postfit[i] = ppxf_tmp[i][7]
             EBV[i] = ppxf_tmp[i][8]
 
-        printStatus.updateDone("Running PPXF in parallel mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in parallel mode", progressbar=False)
 
     if config['GENERAL']['PARALLEL'] == False:
         printStatus.running("Running PPXF in serial mode")
@@ -964,7 +966,7 @@ def worker(chunk, templates):
             mean_results_MC_iter[i,:,:] = mc_results_i["mean_results_MC_iter"]
             mean_results_MC_mean[i,:] = mc_results_i["mean_results_MC_mean"]
             mean_results_MC_err[i,:] = mc_results_i["mean_results_MC_err"]
-        printStatus.updateDone("Running PPXF in serial mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in serial mode", progressbar=False)
 
     print(
         "             Running PPXF on %s spectra took %.2fs using %i cores"

diff --git a/gistPipeline/stellarKinematics/ppxf_kin_wrapper.py b/gistPipeline/stellarKinematics/ppxf_kin_wrapper.py
@@ -1,4 +1,3 @@
-import code
 import logging
 import os
 import time
@@ -12,6 +11,7 @@
 from joblib import Parallel, delayed, dump, load
 from ppxf.ppxf import ppxf
 from printStatus import printStatus
+from tqdm import tqdm
 
 # PHYSICAL CONSTANTS
 C = 299792.458  # km/s
@@ -25,7 +25,6 @@
   ui.adsabs.harvard.edu/?#abs/2017MNRAS.466..798C).
 """
 
-
 def robust_sigma(y, zero=False):
     """
     Biweight estimate of the scale (standard deviation).
@@ -74,7 +73,7 @@ def run_ppxf(
     ui.adsabs.harvard.edu/?#abs/2017MNRAS.466..798C), in order to determine the
     stellar kinematics.
     """
-    printStatus.progressBar(i, nbins, barLength=50)
+    # printStatus.progressBar(i, nbins, barLength=50)
 
     try:
 
@@ -746,11 +745,11 @@ def worker(chunk, templates):
 
         # Use joblib to parallelize the work
         max_nbytes = "1M" # max array size before memory mapping is triggered
-        chunk_size = max(1, nbins // (config["GENERAL"]["NCPU"]))
+        chunk_size = max(1, nbins // ((config["GENERAL"]["NCPU"]) * 10))
         chunks = [range(i, min(i + chunk_size, nbins)) for i in range(0, nbins, chunk_size)]
-        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c"}
-        ppxf_tmp = Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks)
-
+        parallel_configs = {"n_jobs": config["GENERAL"]["NCPU"], "max_nbytes": max_nbytes, "temp_folder": memmap_folder, "mmap_mode": "c", "return_as":"generator"}
+        ppxf_tmp = list(tqdm(Parallel(**parallel_configs)(delayed(worker)(chunk, templates) for chunk in chunks),
+                        total=len(chunks), desc="Processing Chunks"))
         # Flatten the results
         ppxf_tmp = [result for chunk_results in ppxf_tmp for result in chunk_results]
 
@@ -765,7 +764,7 @@ def worker(chunk, templates):
             spectral_mask[i, :] = ppxf_tmp[i][6]
             snr_postfit[i] = ppxf_tmp[i][7]
 
-        printStatus.updateDone("Running PPXF in parallel mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in parallel mode", progressbar=False)
 
     elif config["GENERAL"]["PARALLEL"] == False:
         printStatus.running("Running PPXF in serial mode")
@@ -801,7 +800,7 @@ def worker(chunk, templates):
                 i,
                 optimal_template_comb,
             )
-        printStatus.updateDone("Running PPXF in serial mode", progressbar=True)
+        printStatus.updateDone("Running PPXF in serial mode", progressbar=False)
 
     print(
         "             Running PPXF on %s spectra took %.2fs using %i cores"