Feature/flatten asktell dict values

.github/workflows/basic.yml

@@ -18,28 +18,28 @@ jobs:
                 os: [ubuntu-latest]
                 mpi-version: [mpich]
                 python-version: [3.9, "3.10", "3.11", "3.12"]
-                pydantic-version: ["2.6.4"]
+                pydantic-version: ["2.8.2"]
                 comms-type: [m, l]
                 include:
                     - os: macos-latest
                       python-version: "3.11"
                       mpi-version: mpich
-                      pydantic-version: "2.6.4"
+                      pydantic-version: "2.8.2"
                       comms-type: m
                     - os: macos-latest
                       python-version: "3.11"
                       mpi-version: mpich
-                      pydantic-version: "2.6.4"
+                      pydantic-version: "2.8.2"
                       comms-type: l
                     - os: ubuntu-latest
                       mpi-version: mpich
                       python-version: "3.10"
-                      pydantic-version: "1.10.13"
+                      pydantic-version: "1.10.17"
                       comms-type: m
                     - os: ubuntu-latest
                       mpi-version: mpich
                       python-version: "3.10"
-                      pydantic-version: "1.10.13"
+                      pydantic-version: "1.10.17"
                       comms-type: l
 
         env:
@@ -163,4 +163,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.23.6
+        - uses: crate-ci/typos@v1.24.1

.github/workflows/extra.yml

@@ -12,38 +12,38 @@ jobs:
                 os: [ubuntu-latest]
                 mpi-version: [mpich]
                 python-version: [3.9, "3.10", "3.11", "3.12"]
-                pydantic-version: ["2.6.4"]
+                pydantic-version: ["2.8.2"]
                 comms-type: [m, l]
                 include:
                     - os: macos-latest
                       python-version: 3.11
                       mpi-version: mpich
-                      pydantic-version: "2.6.4"
+                      pydantic-version: "2.8.2"
                       comms-type: m
                     - os: macos-latest
                       python-version: 3.11
                       mpi-version: mpich
-                      pydantic-version: "2.6.4"
+                      pydantic-version: "2.8.2"
                       comms-type: l
                     - os: ubuntu-latest
                       python-version: "3.10"
                       mpi-version: mpich
-                      pydantic-version: "2.6.4"
+                      pydantic-version: "2.8.2"
                       comms-type: t
                     - os: ubuntu-latest
                       mpi-version: "openmpi"
-                      pydantic-version: "2.6.4"
+                      pydantic-version: "2.8.2"
                       python-version: "3.12"
                       comms-type: l
                     - os: ubuntu-latest
                       mpi-version: mpich
                       python-version: "3.10"
-                      pydantic-version: "1.10.13"
+                      pydantic-version: "1.10.17"
                       comms-type: m
                     - os: ubuntu-latest
                       mpi-version: mpich
                       python-version: "3.10"
-                      pydantic-version: "1.10.13"
+                      pydantic-version: "1.10.17"
                       comms-type: l
 
         env:
@@ -157,16 +157,20 @@ jobs:
             pip install git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject
             pip install scikit-build packaging Tasmanian --user
 
-        - name: Install other testing dependencies
+        - name: Install Balsam on Pydantic 1
+          if: matrix.pydantic-version == '1.10.17'
           run: |
-            conda install octave
             conda install pyzmq
-            pip install -r install/testing_requirements.txt
-            pip install -r install/misc_feature_requirements.txt
             git clone https://github.com/argonne-lcf/balsam.git
             sed -i -e "s/pyzmq>=22.1.0,<23.0.0/pyzmq>=23.0.0,<24.0.0/" ./balsam/setup.cfg
             cd balsam; pip install -e .; cd ..
 
+        - name: Install other testing dependencies
+          run: |
+            conda install octave
+            pip install -r install/testing_requirements.txt
+            pip install -r install/misc_feature_requirements.txt
+
             git clone --recurse-submodules -b develop https://github.com/POptUS/IBCDFO.git
             pushd IBCDFO/minq/py/minq5/
             export PYTHONPATH="$PYTHONPATH:$(pwd)"
@@ -224,15 +228,26 @@ jobs:
             rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py
             rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py
 
-        - name: Remove Balsam/Globus-compute tests on Pydantic 2
+        - name: Install redis/proxystore on Pydantic 2
           if: matrix.pydantic-version == '2.6.4'
+          run: |
+            pip install redis
+            pip install proxystore==0.7.0
+
+        - name: Remove proxystore test on Pydantic 1
+          if: matrix.pydantic-version == '1.10.13'
+          run: |
+            rm ./libensemble/tests/regression_tests/test_proxystore_integration.py
+
+        - name: Remove Balsam/Globus-compute tests on Pydantic 2
+          if: matrix.pydantic-version == '2.8.2'
           run: |
             rm ./libensemble/tests/unit_tests/test_ufunc_runners.py
             rm ./libensemble/tests/unit_tests/test_executor_balsam.py
 
         - name: Start Redis
           if: matrix.os == 'ubuntu-latest'
-          uses: supercharge/redis-github-action@1.7.0
+          uses: supercharge/redis-github-action@1.8.0
           with:
             redis-version: 7
 
@@ -261,4 +276,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.23.6
+        - uses: crate-ci/typos@v1.24.1

docs/FAQ.rst

@@ -305,7 +305,7 @@ macOS and Windows Errors
 
 .. _Installing PETSc On Microsoft Windows: https://petsc.org/release/install/windows/#recommended-installation-methods
 .. _option to srun: https://docs.nersc.gov/systems/perlmutter/running-jobs/#single-gpu-tasks-in-parallel
-.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter
+.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/architecture/
 .. _Python multiprocessing docs: https://docs.python.org/3/library/multiprocessing.html
 .. _SDF: https://sdf.slac.stanford.edu/public/doc/#/?id=what-is-the-sdf
 .. _Support: https://libensemble.readthedocs.io/en/main/introduction.html#resources

docs/examples/surmise.rst

@@ -2,15 +2,11 @@ persistent_surmise
 ------------------
 
 Required: Surmise_
-
-Note that currently the github fork https://github.com/mosesyhc/surmise should be used::
-
-    pip install --upgrade git+https://github.com/bandframework/surmise.git@develop
-
-The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an example of the capability to cancel pending simulations.
+The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an
+example of the capability to cancel pending simulations.
 
 .. automodule:: persistent_surmise_calib
   :members:
   :no-undoc-members:
 
-.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html
+.. _Surmise: https://github.com/bandframework/surmise

docs/nitpicky

@@ -43,6 +43,7 @@ py:class libensemble.resources.platforms.Aurora
 py:class libensemble.resources.platforms.GenericROCm
 py:class libensemble.resources.platforms.Crusher
 py:class libensemble.resources.platforms.Frontier
+py:class libensemble.resources.platforms.Perlmutter
 py:class libensemble.resources.platforms.PerlmutterCPU
 py:class libensemble.resources.platforms.PerlmutterGPU
 py:class libensemble.resources.platforms.Polaris

docs/platforms/perlmutter.rst

@@ -190,7 +190,7 @@ See the NERSC Perlmutter_ docs for more information about Perlmutter.
 .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/
 .. _NERSC: https://www.nersc.gov/
 .. _option to srun: https://docs.nersc.gov/systems/perlmutter/running-jobs/#single-gpu-tasks-in-parallel
-.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/
+.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/architecture/
 .. _Python on Perlmutter: https://docs.nersc.gov/development/languages/python/using-python-perlmutter/
 .. _Slurm: https://slurm.schedmd.com/
 .. _video: https://www.youtube.com/watch?v=Av8ctYph7-Y

install/misc_feature_requirements.txt

@@ -1,2 +1 @@
-globus-compute-sdk==2.26.0
-proxystore==0.7.0
+globus-compute-sdk==2.27.0

libensemble/gen_classes/aposmm.py

@@ -14,13 +14,11 @@
     """
 
     def __init__(
-        self, gen_specs: dict = {}, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs
+        self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs
     ) -> None:
         from libensemble.gen_funcs.persistent_aposmm import aposmm
 
         gen_specs["gen_f"] = aposmm
-        if len(kwargs) > 0:  # so user can specify aposmm-specific parameters as kwargs to constructor
-            gen_specs["user"] = kwargs
         if not gen_specs.get("out"):  # gen_specs never especially changes for aposmm even as the problem varies
             n = len(kwargs["lb"]) or len(kwargs["ub"])
             gen_specs["out"] = [
@@ -32,9 +30,8 @@
             ]
             gen_specs["persis_in"] = ["x", "f", "local_pt", "sim_id", "sim_ended", "x_on_cube", "local_min"]
         if not persis_info:
-            persis_info = add_unique_random_streams({}, 4, seed=4321)[1]
-            persis_info["nworkers"] = 4
-        super().__init__(gen_specs, History, persis_info, libE_info)
+            persis_info = add_unique_random_streams({}, 2, seed=4321)[1]
+        super().__init__(History, persis_info, gen_specs, libE_info, **kwargs)
         self.all_local_minima = []
         self.results_idx = 0
         self.last_ask = None
@@ -53,11 +50,11 @@
                 self.all_local_minima.append(self.last_ask[min_idxs])
                 self.last_ask = self.last_ask[~min_idxs]
         if num_points > 0:  # we've been asked for a selection of the last ask
             results = np.copy(
                 self.last_ask[self.results_idx : self.results_idx + num_points]
             )  # if resetting last_ask later, results may point to "None"
             self.results_idx += num_points
             return results
         results = np.copy(self.last_ask)
         self.results = results
         self.last_ask = None

libensemble/gen_classes/sampling.py

@@ -31,10 +31,8 @@ class UniformSample(SampleBase):
     mode by adjusting the allocation function.
     """
 
-    def __init__(self, _, persis_info, gen_specs, libE_info=None):
-        self.persis_info = persis_info
-        self.gen_specs = gen_specs
-        self.libE_info = libE_info
+    def __init__(self, _=[], persis_info={}, gen_specs={}, libE_info=None, **kwargs):
+        super().__init__(_, persis_info, gen_specs, libE_info, **kwargs)
         self._get_user_params(self.gen_specs["user"])
 
     def ask_numpy(self, n_trials):
@@ -57,10 +55,9 @@ class UniformSampleDicts(Generator):
     mode by adjusting the allocation function.
     """
 
-    def __init__(self, _, persis_info, gen_specs, libE_info=None):
-        self.persis_info = persis_info
+    def __init__(self, _, persis_info, gen_specs, libE_info=None, **kwargs):
         self.gen_specs = gen_specs
-        self.libE_info = libE_info
+        self.persis_info = persis_info
         self._get_user_params(self.gen_specs["user"])
 
     def ask(self, n_trials):

libensemble/gen_classes/surmise.py

@@ -14,47 +14,47 @@
     """
 
     def __init__(
-        self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}
+        self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}
     ) -> None:
         from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib
 
         gen_specs["gen_f"] = surmise_calib
         if ("sim_id", int) not in gen_specs["out"]:
             gen_specs["out"].append(("sim_id", int))
-        super().__init__(gen_specs, History, persis_info, libE_info)
+        super().__init__(History, persis_info, gen_specs, libE_info)
         self.sim_id_index = 0
         self.all_cancels = []
 
     def _add_sim_ids(self, array: npt.NDArray) -> npt.NDArray:
         array["sim_id"] = np.arange(self.sim_id_index, self.sim_id_index + len(array))
         self.sim_id_index += len(array)
         return array
 
     def ready_to_be_asked(self) -> bool:
         """Check if the generator has the next batch of points ready."""
         return not self.outbox.empty()
 
     def ask_numpy(self, *args) -> npt.NDArray:
         """Request the next set of points to evaluate, as a NumPy array."""
         output = super().ask_numpy()
         if "cancel_requested" in output.dtype.names:
             cancels = output
             got_cancels_first = True
             self.all_cancels.append(cancels)
         else:
             self.results = self._add_sim_ids(output)
             got_cancels_first = False
         try:
             _, additional = self.outbox.get(timeout=0.2)  # either cancels or new points
             if got_cancels_first:
                 return additional["calc_out"]
             self.all_cancels.append(additional["calc_out"])
             return self.results
         except thread_queue.Empty:
             return self.results
 
     def ask_updates(self) -> List[npt.NDArray]:
         """Request a list of NumPy arrays containing points that should be cancelled by the workflow."""
         cancels = copy.deepcopy(self.all_cancels)
         self.all_cancels = []
         return cancels

libensemble/gen_funcs/persistent_gen_wrapper.py

@@ -1,10 +1,8 @@
 import inspect
 
-import numpy as np
-
 from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG
 from libensemble.tools.persistent_support import PersistentSupport
-from libensemble.utils.misc import np_to_list_dicts
+from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts
 
 
 def persistent_gen_f(H, persis_info, gen_specs, libE_info):
@@ -24,11 +22,7 @@ def persistent_gen_f(H, persis_info, gen_specs, libE_info):
     while tag not in [STOP_TAG, PERSIS_STOP]:
         H_o = gen.ask(b)
         if isinstance(H_o, list):
-            H_o_arr = np.zeros(len(H_o), dtype=gen_specs["out"])
-            for i in range(len(H_o)):
-                for key in H_o[0].keys():
-                    H_o_arr[i][key] = H_o[i][key]
-            H_o = H_o_arr
+            H_o = list_dicts_to_np(H_o)
         tag, Work, calc_in = ps.send_recv(H_o)
         gen.tell(np_to_list_dicts(calc_in))
 

libensemble/generators.py

@@ -8,6 +8,7 @@
 from libensemble.comms.comms import QComm, QCommThread
 from libensemble.executors import Executor
 from libensemble.message_numbers import EVAL_GEN_TAG, PERSIS_STOP
+from libensemble.tools.tools import add_unique_random_streams
 from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts
 
 """
@@ -90,6 +91,17 @@ class LibensembleGenerator(Generator):
     ``ask_numpy/tell_numpy`` methods communicate numpy arrays containing the same data.
     """
 
+    def __init__(
+        self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs
+    ):
+        self.gen_specs = gen_specs
+        if len(kwargs) > 0:  # so user can specify gen-specific parameters as kwargs to constructor
+            self.gen_specs["user"] = kwargs
+        if not persis_info:
+            self.persis_info = add_unique_random_streams({}, 4, seed=4321)[1]
+        else:
+            self.persis_info = persis_info
+
     @abstractmethod
     def ask_numpy(self, num_points: Optional[int] = 0) -> npt.NDArray:
         """Request the next set of points to evaluate, as a NumPy array."""
@@ -105,6 +117,8 @@ def ask(self, num_points: Optional[int] = 0) -> List[dict]:
     def tell(self, results: List[dict]) -> None:
         """Send the results of evaluations to the generator."""
         self.tell_numpy(list_dicts_to_np(results))
+        # Note that although we'd prefer to have a complete dtype available, the gen
+        # doesn't have access to sim_specs["out"] currently.
 
 
 class LibensembleGenThreadInterfacer(LibensembleGenerator):
@@ -113,10 +127,10 @@ class LibensembleGenThreadInterfacer(LibensembleGenerator):
     """
 
     def __init__(
-        self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}
+        self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs
     ) -> None:
+        super().__init__(History, persis_info, gen_specs, libE_info, **kwargs)
         self.gen_f = gen_specs["gen_f"]
-        self.gen_specs = gen_specs
         self.History = History
         self.persis_info = persis_info
         self.libE_info = libE_info

libensemble/tests/regression_tests/test_asktell_surmise.py

@@ -80,7 +80,7 @@
     }
 
     persis_info = add_unique_random_streams({}, 5)
-    surmise = Surmise(gen_specs, persis_info=persis_info[1])  # we add sim_id as a field to gen_specs["out"]
+    surmise = Surmise(gen_specs=gen_specs, persis_info=persis_info[1])  # we add sim_id as a field to gen_specs["out"]
     surmise.setup()
 
     initial_sample = surmise.ask()

libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py

@@ -126,7 +126,7 @@
     }
 
     persis_info = add_unique_random_streams({}, nworkers + 1)
-    gen_specs["generator"] = Surmise(gen_specs, persis_info=persis_info)
+    gen_specs["generator"] = Surmise(gen_specs=gen_specs, persis_info=persis_info)
 
     exit_criteria = {"sim_max": max_evals}
 

libensemble/tests/regression_tests/test_proxystore_integration.py

@@ -11,11 +11,10 @@
 """
 
 # Do not change these lines - they are parsed by run-tests.sh
-# TESTSUITE_COMMS: local
+# TESTSUITE_COMMS: local mpi
 # TESTSUITE_NPROCS: 4
 # TESTSUITE_OS_SKIP: OSX WIN
 # TESTSUITE_EXTRA: true
-# TESTSUITE_EXCLUDE: true
 
 from pathlib import Path
 
@@ -39,7 +38,7 @@ def insert_proxy(H0):
     )
 
     store = get_store("my-store")
-    picture = Path("libE_logo.png").read_bytes()
+    picture = Path("libE_logo.png").absolute().read_bytes()
     proxy = store.proxy(picture)
     for i in range(len(H0)):
         H0[i]["proxy"] = proxy

libensemble/tests/run-tests.sh

@@ -154,7 +154,7 @@ cleanup() {
     filelist=(nodelist_*);              [ -e ${filelist[0]} ] && rm nodelist_*
     filelist=(x_*.txt y_*.txt);         [ -e ${filelist[0]} ] && rm x_*.txt y_*.txt
     filelist=(opt_*.txt_flag);          [ -e ${filelist[0]} ] && rm opt_*.txt_flag
-    filelist=(*.png);                   [ -e ${filelist[0]} ] && rm *.png
+    filelist=(logo_id*.png);            [ -e ${filelist[0]} ] && rm logo_id*.png
   done
   cd $THISDIR
 }