From 5acf6c7860860223568f33d4e3281d0c20a758c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 19:28:41 +0000 Subject: [PATCH 01/34] Bump supercharge/redis-github-action from 1.7.0 to 1.8.0 Bumps [supercharge/redis-github-action](https://github.com/supercharge/redis-github-action) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/supercharge/redis-github-action/releases) - [Changelog](https://github.com/supercharge/redis-github-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/supercharge/redis-github-action/compare/1.7.0...1.8.0) --- updated-dependencies: - dependency-name: supercharge/redis-github-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 4f7538a76..c75ba787a 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -231,7 +231,7 @@ jobs: - name: Start Redis if: matrix.os == 'ubuntu-latest' - uses: supercharge/redis-github-action@1.7.0 + uses: supercharge/redis-github-action@1.8.0 with: redis-version: 7 From 748e16b61cfde2adac034b90c4d87c2a8f562076 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 15 Aug 2024 11:15:29 -0500 Subject: [PATCH 02/34] Fixing broken link --- docs/FAQ.rst | 2 +- docs/platforms/perlmutter.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 37631333d..2397da52f 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -305,7 +305,7 @@ macOS and Windows Errors .. _Installing PETSc On Microsoft Windows: https://petsc.org/release/install/windows/#recommended-installation-methods .. _option to srun: https://docs.nersc.gov/systems/perlmutter/running-jobs/#single-gpu-tasks-in-parallel -.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter +.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/architecture/ .. _Python multiprocessing docs: https://docs.python.org/3/library/multiprocessing.html .. _SDF: https://sdf.slac.stanford.edu/public/doc/#/?id=what-is-the-sdf .. _Support: https://libensemble.readthedocs.io/en/main/introduction.html#resources diff --git a/docs/platforms/perlmutter.rst b/docs/platforms/perlmutter.rst index a8737cd68..915a647af 100644 --- a/docs/platforms/perlmutter.rst +++ b/docs/platforms/perlmutter.rst @@ -190,7 +190,7 @@ See the NERSC Perlmutter_ docs for more information about Perlmutter. .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ .. _NERSC: https://www.nersc.gov/ .. _option to srun: https://docs.nersc.gov/systems/perlmutter/running-jobs/#single-gpu-tasks-in-parallel -.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/ +.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/architecture/ .. _Python on Perlmutter: https://docs.nersc.gov/development/languages/python/using-python-perlmutter/ .. _Slurm: https://slurm.schedmd.com/ .. _video: https://www.youtube.com/watch?v=Av8ctYph7-Y From aa0a976714d2468288607df4afe0569d01da799c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 15 Aug 2024 14:24:03 -0500 Subject: [PATCH 03/34] nitpicky --- docs/nitpicky | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/nitpicky b/docs/nitpicky index 20a0f851d..8315471a1 100644 --- a/docs/nitpicky +++ b/docs/nitpicky @@ -43,6 +43,7 @@ py:class libensemble.resources.platforms.Aurora py:class libensemble.resources.platforms.GenericROCm py:class libensemble.resources.platforms.Crusher py:class libensemble.resources.platforms.Frontier +py:class libensemble.resources.platforms.Perlmutter py:class libensemble.resources.platforms.PerlmutterCPU py:class libensemble.resources.platforms.PerlmutterGPU py:class libensemble.resources.platforms.Polaris From ee2508e46779a831ef774cf0257e44109a076683 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2024 14:21:15 -0500 Subject: [PATCH 04/34] initial commit, creating ask/tell gen unit test, base LibensembleGenerator class can set gen_specs.user via kwargs --- libensemble/gen_classes/aposmm.py | 2 -- libensemble/gen_classes/sampling.py | 14 ++++++-------- libensemble/generators.py | 15 ++++++++++++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_classes/aposmm.py b/libensemble/gen_classes/aposmm.py index 8e8fb47f0..36a2bc390 100644 --- a/libensemble/gen_classes/aposmm.py +++ b/libensemble/gen_classes/aposmm.py @@ -19,8 +19,6 @@ def __init__( from libensemble.gen_funcs.persistent_aposmm import aposmm gen_specs["gen_f"] = aposmm - if len(kwargs) > 0: # so user can specify aposmm-specific parameters as kwargs to constructor - gen_specs["user"] = kwargs if not gen_specs.get("out"): # gen_specs never especially changes for aposmm even as the problem varies n = len(kwargs["lb"]) or len(kwargs["ub"]) gen_specs["out"] = [ diff --git a/libensemble/gen_classes/sampling.py b/libensemble/gen_classes/sampling.py index 275624bb9..e7cbc808a 100644 --- a/libensemble/gen_classes/sampling.py +++ b/libensemble/gen_classes/sampling.py @@ -31,11 +31,10 @@ class UniformSample(SampleBase): mode by adjusting the allocation function. """ - def __init__(self, _, persis_info, gen_specs, libE_info=None): - self.persis_info = persis_info - self.gen_specs = gen_specs - self.libE_info = libE_info + def __init__(self, _=[], persis_info={}, gen_specs={}, libE_info=None, **kwargs): + super().__init__(gen_specs, _, persis_info, libE_info, **kwargs) self._get_user_params(self.gen_specs["user"]) + self.gen_specs["out"] = [("x", float, (self.n,))] def ask_numpy(self, n_trials): H_o = np.zeros(n_trials, dtype=self.gen_specs["out"]) @@ -57,11 +56,10 @@ class UniformSampleDicts(Generator): mode by adjusting the allocation function. """ - def __init__(self, _, persis_info, gen_specs, libE_info=None): - self.persis_info = persis_info - self.gen_specs = gen_specs - self.libE_info = libE_info + def __init__(self, _, persis_info, gen_specs, libE_info=None, **kwargs): + super().__init__(_, persis_info, gen_specs, libE_info) self._get_user_params(self.gen_specs["user"]) + self.gen_specs["out"] = [("x", float, (self.n,))] def ask(self, n_trials): H_o = [] diff --git a/libensemble/generators.py b/libensemble/generators.py index 1ee243954..2aee4bacb 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -8,6 +8,7 @@ from libensemble.comms.comms import QComm, QCommThread from libensemble.executors import Executor from libensemble.message_numbers import EVAL_GEN_TAG, PERSIS_STOP +from libensemble.tools.tools import add_unique_random_streams from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts """ @@ -90,6 +91,18 @@ class LibensembleGenerator(Generator): ``ask_numpy/tell_numpy`` methods communicate numpy arrays containing the same data. """ + def __init__( + self, gen_specs: dict = {}, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs + ): + self.gen_specs = gen_specs + if len(kwargs) > 0: # so user can specify gen-specific parameters as kwargs to constructor + self.gen_specs["user"] = kwargs + if not persis_info: + self.persis_info = add_unique_random_streams({}, 4, seed=4321)[1] + self.persis_info["nworkers"] = 4 + else: + self.persis_info = persis_info + @abstractmethod def ask_numpy(self, num_points: Optional[int] = 0) -> npt.NDArray: """Request the next set of points to evaluate, as a NumPy array.""" @@ -115,8 +128,8 @@ class LibensembleGenThreadInterfacer(LibensembleGenerator): def __init__( self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {} ) -> None: + super().__init__(gen_specs, History, persis_info, libE_info) self.gen_f = gen_specs["gen_f"] - self.gen_specs = gen_specs self.History = History self.persis_info = persis_info self.libE_info = libE_info From 070fc6f9f76b5fa25b3d6d84704e55e7389c788e Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2024 15:12:30 -0500 Subject: [PATCH 05/34] add test, arrays become flattened dicts in np_to_list_dicts --- libensemble/tests/unit_tests/test_asktell.py | 36 ++++++++++++++++++++ libensemble/utils/misc.py | 6 +++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 libensemble/tests/unit_tests/test_asktell.py diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py new file mode 100644 index 000000000..0adef408f --- /dev/null +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -0,0 +1,36 @@ +import numpy as np + +from libensemble.tools.tools import add_unique_random_streams + + +def test_asktell_sampling(): + from libensemble.gen_classes.sampling import UniformSample + + persis_info = add_unique_random_streams({}, 5, seed=1234) + gen_specs = { + "out": [("x", float, (2,))], + "user": { + "lb": np.array([-3, -2]), + "ub": np.array([3, 2]), + }, + } + + # Test initialization with libensembley parameters + gen = UniformSample(None, persis_info[1], gen_specs, None) + assert len(gen.ask(10)) == 10 + + # Test initialization gen-specific keyword args + gen = UniformSample(lb=np.array([-3, -2]), ub=np.array([3, 2])) + assert len(gen.ask(10)) == 10 + + import ipdb + + ipdb.set_trace() + + out = gen.ask_numpy(3) # should get numpy arrays, non-flattened + out = gen.ask(3) # needs to get dicts, 2d+ arrays need to be flattened + assert all([len(x) == 2 for x in out]) # np_to_list_dicts is now tested + + +if __name__ == "__main__": + test_asktell_sampling() diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index db73ccf91..7a7704183 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -117,6 +117,10 @@ def np_to_list_dicts(array: npt.NDArray) -> List[dict]: for row in array: new_dict = {} for field in row.dtype.names: - new_dict[field] = row[field] + if len(row[field]) > 1: + for i, x in enumerate(row[field]): + new_dict[field + str(i)] = x + else: + new_dict[field] = row[field] out.append(new_dict) return out From a969f500f58f52609ba954829477cef8041702d9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2024 15:16:36 -0500 Subject: [PATCH 06/34] remove debug statement --- libensemble/tests/unit_tests/test_asktell.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index 0adef408f..6b79060ab 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -23,10 +23,6 @@ def test_asktell_sampling(): gen = UniformSample(lb=np.array([-3, -2]), ub=np.array([3, 2])) assert len(gen.ask(10)) == 10 - import ipdb - - ipdb.set_trace() - out = gen.ask_numpy(3) # should get numpy arrays, non-flattened out = gen.ask(3) # needs to get dicts, 2d+ arrays need to be flattened assert all([len(x) == 2 for x in out]) # np_to_list_dicts is now tested From 6eb5fe86d4edd942fa474c106e9a29eca526cdcf Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2024 17:42:25 -0500 Subject: [PATCH 07/34] additional attempts to unflatten the input dict... --- libensemble/tests/unit_tests/test_asktell.py | 4 ++ libensemble/utils/misc.py | 49 ++++++++++++++------ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index 6b79060ab..c7b43bc02 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -1,6 +1,7 @@ import numpy as np from libensemble.tools.tools import add_unique_random_streams +from libensemble.utils.misc import list_dicts_to_np def test_asktell_sampling(): @@ -27,6 +28,9 @@ def test_asktell_sampling(): out = gen.ask(3) # needs to get dicts, 2d+ arrays need to be flattened assert all([len(x) == 2 for x in out]) # np_to_list_dicts is now tested + # now we test list_dicts_to_np directly + out = list_dicts_to_np(out) + if __name__ == "__main__": test_asktell_sampling() diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index 7a7704183..e8c2e235b 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -81,18 +81,15 @@ def specs_checker_setattr(obj, key, value): obj.__dict__[key] = value -def _copy_data(array, list_dicts): - for i, entry in enumerate(list_dicts): - for field in entry.keys(): - array[field][i] = entry[field] - return array +def _decide_dtype(name, entry, size): + if size == 1: + return (name, type(entry)) + else: + return (name, type(entry), (size,)) -def _decide_dtype(name, entry): - if hasattr(entry, "shape") and len(entry.shape): # numpy type - return (name, entry.dtype, entry.shape) - else: - return (name, type(entry)) +def _combine_names(names): + return list(set(i[:-1] if i[-1].isdigit() else i for i in names)) def list_dicts_to_np(list_dicts: list) -> npt.NDArray: @@ -100,14 +97,38 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: return None first = list_dicts[0] - new_dtype_names = [i for i in first.keys()] + new_dtype_names = _combine_names([i for i in first.keys()]) new_dtype = [] - for i, entry in enumerate(first.values()): # must inspect values to get presumptive types + combinable_names = [] + for name in new_dtype_names: + combinable_names.append([i for i in first.keys() if i.startswith(name)]) + + for i, entry in enumerate(combinable_names): # must inspect values to get presumptive types name = new_dtype_names[i] - new_dtype.append(_decide_dtype(name, entry)) + size = len(combinable_names[i]) + new_dtype.append(_decide_dtype(name, first[entry[0]], size)) out = np.zeros(len(list_dicts), dtype=new_dtype) - return _copy_data(out, list_dicts) + + # good lord, this is ugly + # for names_group_idx, entry in enumerate(combinable_names): + # for input_dict in list_dicts: + # for l in range(len(input_dict)): + # for name_idx, src_key in enumerate(entry): + # out[new_dtype_names[names_group_idx]][name_idx][l] = input_dict[src_key] + + for name in new_dtype_names: + for i, input_dict in enumerate(list_dicts): + for j, value in enumerate(input_dict.values()): + out[name][j][i] = value + + [ + {"x0": -1.3315287487797274, "x1": -1.1102419596798931}, + {"x0": 2.2035749254093417, "x1": -0.04551905560134939}, + {"x0": -1.043550345357007, "x1": -0.853671651707665}, + ] + + return out def np_to_list_dicts(array: npt.NDArray) -> List[dict]: From 12612744cf1633dc8be36bb8ac183fc54d75d1f2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2024 10:54:19 -0500 Subject: [PATCH 08/34] fix index ordering, cleanup/complete tentatively unit test --- libensemble/tests/unit_tests/test_asktell.py | 14 ++++++--- libensemble/utils/misc.py | 33 +++++++------------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index c7b43bc02..660e19ae8 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -4,7 +4,7 @@ from libensemble.utils.misc import list_dicts_to_np -def test_asktell_sampling(): +def test_asktell_sampling_and_utils(): from libensemble.gen_classes.sampling import UniformSample persis_info = add_unique_random_streams({}, 5, seed=1234) @@ -24,13 +24,19 @@ def test_asktell_sampling(): gen = UniformSample(lb=np.array([-3, -2]), ub=np.array([3, 2])) assert len(gen.ask(10)) == 10 - out = gen.ask_numpy(3) # should get numpy arrays, non-flattened + out_np = gen.ask_numpy(3) # should get numpy arrays, non-flattened out = gen.ask(3) # needs to get dicts, 2d+ arrays need to be flattened assert all([len(x) == 2 for x in out]) # np_to_list_dicts is now tested # now we test list_dicts_to_np directly - out = list_dicts_to_np(out) + out_np = list_dicts_to_np(out) + + # check combined values resemble flattened list-of-dicts values + assert out_np.dtype.names == ("x",) + for i, entry in enumerate(out): + for j, value in enumerate(entry.values()): + assert value == out_np["x"][i][j] if __name__ == "__main__": - test_asktell_sampling() + test_asktell_sampling_and_utils() diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index e8c2e235b..a5de08695 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -81,14 +81,17 @@ def specs_checker_setattr(obj, key, value): obj.__dict__[key] = value -def _decide_dtype(name, entry, size): +def _decide_dtype(name: str, entry, size: int) -> tuple: if size == 1: return (name, type(entry)) else: return (name, type(entry), (size,)) -def _combine_names(names): +def _combine_names(names: list) -> list: + """combine fields with same name *except* for final digit""" + # how many final digits could possibly be in each name? + # do we have to iterate through negative-indexes until we reach a non-digit? return list(set(i[:-1] if i[-1].isdigit() else i for i in names)) @@ -96,37 +99,25 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: if list_dicts is None: return None - first = list_dicts[0] - new_dtype_names = _combine_names([i for i in first.keys()]) - new_dtype = [] - combinable_names = [] + first = list_dicts[0] # for determining dtype of output np array + new_dtype_names = _combine_names([i for i in first.keys()]) # -> ['x', 'y'] + combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2']] for name in new_dtype_names: combinable_names.append([i for i in first.keys() if i.startswith(name)]) - for i, entry in enumerate(combinable_names): # must inspect values to get presumptive types + new_dtype = [] + + for i, entry in enumerate(combinable_names): name = new_dtype_names[i] size = len(combinable_names[i]) new_dtype.append(_decide_dtype(name, first[entry[0]], size)) out = np.zeros(len(list_dicts), dtype=new_dtype) - # good lord, this is ugly - # for names_group_idx, entry in enumerate(combinable_names): - # for input_dict in list_dicts: - # for l in range(len(input_dict)): - # for name_idx, src_key in enumerate(entry): - # out[new_dtype_names[names_group_idx]][name_idx][l] = input_dict[src_key] - for name in new_dtype_names: for i, input_dict in enumerate(list_dicts): for j, value in enumerate(input_dict.values()): - out[name][j][i] = value - - [ - {"x0": -1.3315287487797274, "x1": -1.1102419596798931}, - {"x0": 2.2035749254093417, "x1": -0.04551905560134939}, - {"x0": -1.043550345357007, "x1": -0.853671651707665}, - ] + out[name][i][j] = value return out From d960b960bf70b11c11e8f1a203b0a7c1f0a62320 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2024 12:49:44 -0500 Subject: [PATCH 09/34] passthrough kwargs to superclasses, try to handle empty lists for single-dim fields --- libensemble/gen_classes/aposmm.py | 2 +- libensemble/generators.py | 4 ++-- libensemble/utils/misc.py | 13 ++++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/libensemble/gen_classes/aposmm.py b/libensemble/gen_classes/aposmm.py index 36a2bc390..17caa6f4c 100644 --- a/libensemble/gen_classes/aposmm.py +++ b/libensemble/gen_classes/aposmm.py @@ -32,7 +32,7 @@ def __init__( if not persis_info: persis_info = add_unique_random_streams({}, 4, seed=4321)[1] persis_info["nworkers"] = 4 - super().__init__(gen_specs, History, persis_info, libE_info) + super().__init__(gen_specs, History, persis_info, libE_info, **kwargs) self.all_local_minima = [] self.results_idx = 0 self.last_ask = None diff --git a/libensemble/generators.py b/libensemble/generators.py index 2aee4bacb..b61ba1099 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -126,9 +126,9 @@ class LibensembleGenThreadInterfacer(LibensembleGenerator): """ def __init__( - self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {} + self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs ) -> None: - super().__init__(gen_specs, History, persis_info, libE_info) + super().__init__(gen_specs, History, persis_info, libE_info, **kwargs) self.gen_f = gen_specs["gen_f"] self.History = History self.persis_info = persis_info diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index a5de08695..2de1c841b 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -82,7 +82,7 @@ def specs_checker_setattr(obj, key, value): def _decide_dtype(name: str, entry, size: int) -> tuple: - if size == 1: + if size == 1 or not size: return (name, type(entry)) else: return (name, type(entry), (size,)) @@ -101,16 +101,19 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: first = list_dicts[0] # for determining dtype of output np array new_dtype_names = _combine_names([i for i in first.keys()]) # -> ['x', 'y'] - combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2']] + combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2'], []] for name in new_dtype_names: - combinable_names.append([i for i in first.keys() if i.startswith(name)]) + combinable_names.append([i for i in first.keys() if i[:-1] == name]) new_dtype = [] for i, entry in enumerate(combinable_names): name = new_dtype_names[i] size = len(combinable_names[i]) - new_dtype.append(_decide_dtype(name, first[entry[0]], size)) + if len(entry): # combinable names detected, e.g. x0, x1 + new_dtype.append(_decide_dtype(name, first[entry[0]], size)) + else: # only a single name, e.g. local_pt + new_dtype.append(_decide_dtype(name, first[name], size)) out = np.zeros(len(list_dicts), dtype=new_dtype) @@ -129,7 +132,7 @@ def np_to_list_dicts(array: npt.NDArray) -> List[dict]: for row in array: new_dict = {} for field in row.dtype.names: - if len(row[field]) > 1: + if hasattr(row[field], "__len__") and len(row[field]) > 1: for i, x in enumerate(row[field]): new_dict[field + str(i)] = x else: From 3ce0ca2997a793a42f4062baa6c44a76483de221 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2024 13:19:06 -0500 Subject: [PATCH 10/34] better handling of multi-dim and single-dim output-array item assignment from input list of dicts --- libensemble/utils/misc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index 2de1c841b..e6b810ce0 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -117,10 +117,13 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: out = np.zeros(len(list_dicts), dtype=new_dtype) - for name in new_dtype_names: - for i, input_dict in enumerate(list_dicts): - for j, value in enumerate(input_dict.values()): - out[name][i][j] = value + for i, group in enumerate(combinable_names): + new_dtype_name = new_dtype_names[i] + for j, input_dict in enumerate(list_dicts): + if not len(group): + out[new_dtype_name][j] = input_dict[new_dtype_name] + else: + out[new_dtype_name][j] = tuple([input_dict[name] for name in group]) return out From 09cb4a68d2d9624a35d582ed5c14132d51e27792 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2024 13:21:39 -0500 Subject: [PATCH 11/34] comments --- libensemble/utils/misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index e6b810ce0..878bc1dff 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -120,9 +120,9 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: for i, group in enumerate(combinable_names): new_dtype_name = new_dtype_names[i] for j, input_dict in enumerate(list_dicts): - if not len(group): + if not len(group): # only a single name, e.g. local_pt out[new_dtype_name][j] = input_dict[new_dtype_name] - else: + else: # combinable names detected, e.g. x0, x1 out[new_dtype_name][j] = tuple([input_dict[name] for name in group]) return out From 601f02c2463629a2a4d88abc0f4a707d4f438122 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2024 13:58:49 -0500 Subject: [PATCH 12/34] adjust persistent_gen_wrapper, fix UniformSampleDicts --- libensemble/gen_classes/sampling.py | 3 ++- libensemble/gen_funcs/persistent_gen_wrapper.py | 10 ++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/libensemble/gen_classes/sampling.py b/libensemble/gen_classes/sampling.py index e7cbc808a..d11998e11 100644 --- a/libensemble/gen_classes/sampling.py +++ b/libensemble/gen_classes/sampling.py @@ -57,7 +57,8 @@ class UniformSampleDicts(Generator): """ def __init__(self, _, persis_info, gen_specs, libE_info=None, **kwargs): - super().__init__(_, persis_info, gen_specs, libE_info) + self.gen_specs = gen_specs + self.persis_info = persis_info self._get_user_params(self.gen_specs["user"]) self.gen_specs["out"] = [("x", float, (self.n,))] diff --git a/libensemble/gen_funcs/persistent_gen_wrapper.py b/libensemble/gen_funcs/persistent_gen_wrapper.py index 2ad862864..7fd01ec4d 100644 --- a/libensemble/gen_funcs/persistent_gen_wrapper.py +++ b/libensemble/gen_funcs/persistent_gen_wrapper.py @@ -1,10 +1,8 @@ import inspect -import numpy as np - from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG from libensemble.tools.persistent_support import PersistentSupport -from libensemble.utils.misc import np_to_list_dicts +from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts def persistent_gen_f(H, persis_info, gen_specs, libE_info): @@ -24,11 +22,7 @@ def persistent_gen_f(H, persis_info, gen_specs, libE_info): while tag not in [STOP_TAG, PERSIS_STOP]: H_o = gen.ask(b) if isinstance(H_o, list): - H_o_arr = np.zeros(len(H_o), dtype=gen_specs["out"]) - for i in range(len(H_o)): - for key in H_o[0].keys(): - H_o_arr[i][key] = H_o[i][key] - H_o = H_o_arr + H_o = list_dicts_to_np(H_o) tag, Work, calc_in = ps.send_recv(H_o) gen.tell(np_to_list_dicts(calc_in)) From 8b99cc110c316cfc6f3a32e8510bfe0d3f46f0dd Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 20 Aug 2024 09:51:46 -0500 Subject: [PATCH 13/34] bump pydantic versions, really trying to resolve warnings --- .github/workflows/basic.yml | 10 +++++----- .github/workflows/extra.yml | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index f9a04caa1..d63a26e92 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -18,28 +18,28 @@ jobs: os: [ubuntu-latest] mpi-version: [mpich] python-version: [3.9, "3.10", "3.11", "3.12"] - pydantic-version: ["2.6.4"] + pydantic-version: ["2.8.2"] comms-type: [m, l] include: - os: macos-latest python-version: "3.11" mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: m - os: macos-latest python-version: "3.11" mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: l - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: m - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: l env: diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 4f7538a76..a0352c0e1 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -12,38 +12,38 @@ jobs: os: [ubuntu-latest] mpi-version: [mpich] python-version: [3.9, "3.10", "3.11", "3.12"] - pydantic-version: ["2.6.4"] + pydantic-version: ["2.8.2"] comms-type: [m, l] include: - os: macos-latest python-version: 3.11 mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: m - os: macos-latest python-version: 3.11 mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: l - os: ubuntu-latest python-version: "3.10" mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: t - os: ubuntu-latest mpi-version: "openmpi" - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" python-version: "3.12" comms-type: l - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: m - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: l env: @@ -224,7 +224,7 @@ jobs: rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py - name: Remove Balsam/Globus-compute tests on Pydantic 2 - if: matrix.pydantic-version == '2.6.4' + if: matrix.pydantic-version == '2.8.2' run: | rm ./libensemble/tests/unit_tests/test_ufunc_runners.py rm ./libensemble/tests/unit_tests/test_executor_balsam.py From b0c31071b0e66721cc8197d7eb98cf7afdc6287e Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 20 Aug 2024 10:21:04 -0500 Subject: [PATCH 14/34] only install balsam on pydantic 1 --- .github/workflows/extra.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index a0352c0e1..7dc6a349f 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -157,16 +157,20 @@ jobs: pip install git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject pip install scikit-build packaging Tasmanian --user - - name: Install other testing dependencies + - name: Install Balsam on Pydantic 1 + if: matrix.pydantic-version == '1.10.17' run: | - conda install octave conda install pyzmq - pip install -r install/testing_requirements.txt - pip install -r install/misc_feature_requirements.txt git clone https://github.com/argonne-lcf/balsam.git sed -i -e "s/pyzmq>=22.1.0,<23.0.0/pyzmq>=23.0.0,<24.0.0/" ./balsam/setup.cfg cd balsam; pip install -e .; cd .. + - name: Install other testing dependencies + run: | + conda install octave + pip install -r install/testing_requirements.txt + pip install -r install/misc_feature_requirements.txt + git clone --recurse-submodules -b develop https://github.com/POptUS/IBCDFO.git pushd IBCDFO/minq/py/minq5/ export PYTHONPATH="$PYTHONPATH:$(pwd)" From 2aac3fa74de9c30e6637f5a1bb15ed6b5def01ed Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 20 Aug 2024 13:44:39 -0500 Subject: [PATCH 15/34] various fixes to run the proxystore test on extra-ci --- .github/workflows/extra.yml | 11 +++++++++++ install/misc_feature_requirements.txt | 1 - .../regression_tests/test_proxystore_integration.py | 5 ++--- libensemble/tests/run-tests.sh | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 4f7538a76..111fa708e 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -223,6 +223,17 @@ jobs: rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py + - name: Install redis/proxystore on Pydantic 2 + if: matrix.pydantic-version == '2.6.4' + run: | + pip install redis + pip install proxystore==0.7.0 + + - name: Remove proxystore test on Pydantic 1 + if: matrix.pydantic-version == '1.10.13' + run: | + rm ./libensemble/tests/regression_tests/test_proxystore_integration.py + - name: Remove Balsam/Globus-compute tests on Pydantic 2 if: matrix.pydantic-version == '2.6.4' run: | diff --git a/install/misc_feature_requirements.txt b/install/misc_feature_requirements.txt index 05416fde6..7d60b228b 100644 --- a/install/misc_feature_requirements.txt +++ b/install/misc_feature_requirements.txt @@ -1,2 +1 @@ globus-compute-sdk==2.26.0 -proxystore==0.7.0 diff --git a/libensemble/tests/regression_tests/test_proxystore_integration.py b/libensemble/tests/regression_tests/test_proxystore_integration.py index 998a45ab9..a900f876d 100644 --- a/libensemble/tests/regression_tests/test_proxystore_integration.py +++ b/libensemble/tests/regression_tests/test_proxystore_integration.py @@ -11,11 +11,10 @@ """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: local mpi # TESTSUITE_NPROCS: 4 # TESTSUITE_OS_SKIP: OSX WIN # TESTSUITE_EXTRA: true -# TESTSUITE_EXCLUDE: true from pathlib import Path @@ -39,7 +38,7 @@ def insert_proxy(H0): ) store = get_store("my-store") - picture = Path("libE_logo.png").read_bytes() + picture = Path("libE_logo.png").absolute().read_bytes() proxy = store.proxy(picture) for i in range(len(H0)): H0[i]["proxy"] = proxy diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 6cdb5b143..a09eebbd4 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -154,7 +154,7 @@ cleanup() { filelist=(nodelist_*); [ -e ${filelist[0]} ] && rm nodelist_* filelist=(x_*.txt y_*.txt); [ -e ${filelist[0]} ] && rm x_*.txt y_*.txt filelist=(opt_*.txt_flag); [ -e ${filelist[0]} ] && rm opt_*.txt_flag - filelist=(*.png); [ -e ${filelist[0]} ] && rm *.png + filelist=(logo_id*.png); [ -e ${filelist[0]} ] && rm logo_id*.png done cd $THISDIR } From 6733fe5cd30676c8d713b536429292e1cbaf8a61 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2024 10:28:54 -0500 Subject: [PATCH 16/34] fix ordering of parameters in implemented ask/tell classes and parent classes, fix aposmm unit test --- libensemble/gen_classes/aposmm.py | 4 ++-- libensemble/gen_classes/sampling.py | 2 +- libensemble/gen_classes/surmise.py | 4 ++-- libensemble/generators.py | 6 +++--- .../tests/unit_tests/RENAME_test_persistent_aposmm.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_classes/aposmm.py b/libensemble/gen_classes/aposmm.py index 17caa6f4c..d49832730 100644 --- a/libensemble/gen_classes/aposmm.py +++ b/libensemble/gen_classes/aposmm.py @@ -14,7 +14,7 @@ class APOSMM(LibensembleGenThreadInterfacer): """ def __init__( - self, gen_specs: dict = {}, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs ) -> None: from libensemble.gen_funcs.persistent_aposmm import aposmm @@ -32,7 +32,7 @@ def __init__( if not persis_info: persis_info = add_unique_random_streams({}, 4, seed=4321)[1] persis_info["nworkers"] = 4 - super().__init__(gen_specs, History, persis_info, libE_info, **kwargs) + super().__init__(History, persis_info, gen_specs, libE_info, **kwargs) self.all_local_minima = [] self.results_idx = 0 self.last_ask = None diff --git a/libensemble/gen_classes/sampling.py b/libensemble/gen_classes/sampling.py index d11998e11..dd347db51 100644 --- a/libensemble/gen_classes/sampling.py +++ b/libensemble/gen_classes/sampling.py @@ -32,7 +32,7 @@ class UniformSample(SampleBase): """ def __init__(self, _=[], persis_info={}, gen_specs={}, libE_info=None, **kwargs): - super().__init__(gen_specs, _, persis_info, libE_info, **kwargs) + super().__init__(_, persis_info, gen_specs, libE_info, **kwargs) self._get_user_params(self.gen_specs["user"]) self.gen_specs["out"] = [("x", float, (self.n,))] diff --git a/libensemble/gen_classes/surmise.py b/libensemble/gen_classes/surmise.py index 3e1810f98..b62cd20dc 100644 --- a/libensemble/gen_classes/surmise.py +++ b/libensemble/gen_classes/surmise.py @@ -14,14 +14,14 @@ class Surmise(LibensembleGenThreadInterfacer): """ def __init__( - self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {} + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {} ) -> None: from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib gen_specs["gen_f"] = surmise_calib if ("sim_id", int) not in gen_specs["out"]: gen_specs["out"].append(("sim_id", int)) - super().__init__(gen_specs, History, persis_info, libE_info) + super().__init__(History, persis_info, gen_specs, libE_info) self.sim_id_index = 0 self.all_cancels = [] diff --git a/libensemble/generators.py b/libensemble/generators.py index b61ba1099..5e9d957b4 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -92,7 +92,7 @@ class LibensembleGenerator(Generator): """ def __init__( - self, gen_specs: dict = {}, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs ): self.gen_specs = gen_specs if len(kwargs) > 0: # so user can specify gen-specific parameters as kwargs to constructor @@ -126,9 +126,9 @@ class LibensembleGenThreadInterfacer(LibensembleGenerator): """ def __init__( - self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs ) -> None: - super().__init__(gen_specs, History, persis_info, libE_info, **kwargs) + super().__init__(History, persis_info, gen_specs, libE_info, **kwargs) self.gen_f = gen_specs["gen_f"] self.History = History self.persis_info = persis_info diff --git a/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py b/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py index 11cad7c63..9bc097a18 100644 --- a/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py +++ b/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py @@ -203,7 +203,7 @@ def test_asktell_with_persistent_aposmm(): }, } - my_APOSMM = APOSMM(gen_specs) + my_APOSMM = APOSMM(gen_specs=gen_specs) my_APOSMM.setup() initial_sample = my_APOSMM.ask(100) @@ -211,7 +211,7 @@ def test_asktell_with_persistent_aposmm(): eval_max = 2000 for point in initial_sample: - point["f"] = six_hump_camel_func(point["x"]) + point["f"] = six_hump_camel_func(np.array([point["x0"], point["x1"]])) total_evals += 1 my_APOSMM.tell(initial_sample) @@ -225,7 +225,7 @@ def test_asktell_with_persistent_aposmm(): for m in detected_minima: potential_minima.append(m) for point in sample: - point["f"] = six_hump_camel_func(point["x"]) + point["f"] = six_hump_camel_func(np.array([point["x0"], point["x1"]])) total_evals += 1 my_APOSMM.tell(sample) H, persis_info, exit_code = my_APOSMM.final_tell(list_dicts_to_np(sample)) # final_tell currently requires numpy From 74661007e1a271f6a427c747ab9ac2164cf2be77 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2024 14:55:30 -0500 Subject: [PATCH 17/34] better detecting of combinable names, by stripping out the numeric suffix, instead of just checking if last char is digit. better decide output numpy array type for strings --- libensemble/tests/unit_tests/test_asktell.py | 29 ++++++++++++++++++++ libensemble/utils/misc.py | 17 ++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index 660e19ae8..6ff789356 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -38,5 +38,34 @@ def test_asktell_sampling_and_utils(): assert value == out_np["x"][i][j] +def test_additional_converts(): + from libensemble.utils.misc import list_dicts_to_np + + # test list_dicts_to_np on a weirdly formatted dictionary + out_np = list_dicts_to_np( + [ + { + "x0": "abcd", + "x1": "efgh", + "y": 56, + "z0": 1, + "z1": 2, + "z2": 3, + "z3": 4, + "z4": 5, + "z5": 6, + "z6": 7, + "z7": 8, + "z8": 9, + "z9": 10, + "z10": 11, + } + ] + ) + + assert all([i in ("x", "y", "z") for i in out_np.dtype.names]) + + if __name__ == "__main__": test_asktell_sampling_and_utils() + test_additional_converts() diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index 878bc1dff..f7b2b3737 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -82,17 +82,21 @@ def specs_checker_setattr(obj, key, value): def _decide_dtype(name: str, entry, size: int) -> tuple: + if isinstance(entry, str): + output_type = "U" + str(len(entry) + 1) + else: + output_type = type(entry) if size == 1 or not size: - return (name, type(entry)) + return (name, output_type) else: - return (name, type(entry), (size,)) + return (name, output_type, (size,)) def _combine_names(names: list) -> list: - """combine fields with same name *except* for final digit""" + """combine fields with same name *except* for final digits""" # how many final digits could possibly be in each name? # do we have to iterate through negative-indexes until we reach a non-digit? - return list(set(i[:-1] if i[-1].isdigit() else i for i in names)) + return list(set(i.rstrip("0123456789") for i in names)) def list_dicts_to_np(list_dicts: list) -> npt.NDArray: @@ -103,7 +107,8 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: new_dtype_names = _combine_names([i for i in first.keys()]) # -> ['x', 'y'] combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2'], []] for name in new_dtype_names: - combinable_names.append([i for i in first.keys() if i[:-1] == name]) + combinable_group = [i for i in first.keys() if i.rstrip("0123456789") == name] + combinable_names.append(combinable_group) new_dtype = [] @@ -120,7 +125,7 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: for i, group in enumerate(combinable_names): new_dtype_name = new_dtype_names[i] for j, input_dict in enumerate(list_dicts): - if not len(group): # only a single name, e.g. local_pt + if len(group) == 1: # only a single name, e.g. local_pt out[new_dtype_name][j] = input_dict[new_dtype_name] else: # combinable names detected, e.g. x0, x1 out[new_dtype_name][j] = tuple([input_dict[name] for name in group]) From a612692e8c2f25c84ead1ddd30e6a6f258553ad7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 19:55:54 +0000 Subject: [PATCH 18/34] Bump globus-compute-sdk from 2.26.0 to 2.27.0 Bumps [globus-compute-sdk](https://github.com/globus/globus-compute) from 2.26.0 to 2.27.0. - [Release notes](https://github.com/globus/globus-compute/releases) - [Changelog](https://github.com/globus/globus-compute/blob/main/docs/changelog.rst) - [Commits](https://github.com/globus/globus-compute/compare/2.26.0...2.27.0) --- updated-dependencies: - dependency-name: globus-compute-sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- install/misc_feature_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/misc_feature_requirements.txt b/install/misc_feature_requirements.txt index 7d60b228b..b72c73c16 100644 --- a/install/misc_feature_requirements.txt +++ b/install/misc_feature_requirements.txt @@ -1 +1 @@ -globus-compute-sdk==2.26.0 +globus-compute-sdk==2.27.0 From 751de5e8c2c1849f585ed38ccc9c65313b9260ba Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2024 15:51:52 -0500 Subject: [PATCH 19/34] deal with keys that end with integers, but aren't similar to any other keys. e.g. {"co2": 12} --- libensemble/tests/unit_tests/test_asktell.py | 3 +- libensemble/utils/misc.py | 30 +++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index 6ff789356..dbdc4148d 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -59,11 +59,12 @@ def test_additional_converts(): "z8": 9, "z9": 10, "z10": 11, + "a0": "B", } ] ) - assert all([i in ("x", "y", "z") for i in out_np.dtype.names]) + assert all([i in ("x", "y", "z", "a0") for i in out_np.dtype.names]) if __name__ == "__main__": diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index f7b2b3737..659a97440 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -94,9 +94,18 @@ def _decide_dtype(name: str, entry, size: int) -> tuple: def _combine_names(names: list) -> list: """combine fields with same name *except* for final digits""" - # how many final digits could possibly be in each name? - # do we have to iterate through negative-indexes until we reach a non-digit? - return list(set(i.rstrip("0123456789") for i in names)) + + out_names = [] + stripped = list(i.rstrip("0123456789") for i in names) # ['x', 'x', y', 'z', 'a'] + for name in names: + stripped_name = name.rstrip("0123456789") + if stripped.count(stripped_name) > 1: # if name appears >= 1, will combine, don't keep int suffix + out_names.append(stripped_name) + else: + out_names.append(name) # name appears once, keep integer suffix, e.g. "co2" + + # intending [x, y, z, a0] from [x0, x1, y, z0, z1, z2, z3, a0] + return list(set(out_names)) def list_dicts_to_np(list_dicts: list) -> npt.NDArray: @@ -105,20 +114,21 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: first = list_dicts[0] # for determining dtype of output np array new_dtype_names = _combine_names([i for i in first.keys()]) # -> ['x', 'y'] - combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2'], []] - for name in new_dtype_names: + combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2'], ['z']] + for name in new_dtype_names: # is this a necessary search over the keys again? we did it earlier... combinable_group = [i for i in first.keys() if i.rstrip("0123456789") == name] - combinable_names.append(combinable_group) + if len(combinable_group) > 1: # multiple similar names, e.g. x0, x1 + combinable_names.append(combinable_group) + else: # single name, e.g. local_pt, a0 *AS LONG AS THERE ISNT AN A1* + combinable_names.append([name]) new_dtype = [] + # another loop over names, there's probably a more elegant way, but my brain is fried for i, entry in enumerate(combinable_names): name = new_dtype_names[i] size = len(combinable_names[i]) - if len(entry): # combinable names detected, e.g. x0, x1 - new_dtype.append(_decide_dtype(name, first[entry[0]], size)) - else: # only a single name, e.g. local_pt - new_dtype.append(_decide_dtype(name, first[name], size)) + new_dtype.append(_decide_dtype(name, first[entry[0]], size)) out = np.zeros(len(list_dicts), dtype=new_dtype) From be0af7c9edce1394e140086b171e7f925918f32b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 19:20:08 +0000 Subject: [PATCH 20/34] Bump crate-ci/typos from 1.23.6 to 1.23.7 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.23.6 to 1.23.7. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/v1.23.6...v1.23.7) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/basic.yml | 2 +- .github/workflows/extra.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index d63a26e92..ef6f92eb9 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -163,4 +163,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.23.6 + - uses: crate-ci/typos@v1.23.7 diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 08e3b654d..f493c08f2 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -275,4 +275,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.23.6 + - uses: crate-ci/typos@v1.23.7 From 18e70794cfd24bd6f90c3ee47029651881833003 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 22 Aug 2024 14:49:21 -0500 Subject: [PATCH 21/34] keyword assignment of gen_specs to Surmise --- .../test_persistent_surmise_killsims_asktell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py index 842573de9..9071e80d4 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py @@ -126,7 +126,7 @@ } persis_info = add_unique_random_streams({}, nworkers + 1) - gen_specs["generator"] = Surmise(gen_specs, persis_info=persis_info) + gen_specs["generator"] = Surmise(gen_specs=gen_specs, persis_info=persis_info) exit_criteria = {"sim_max": max_evals} From a34d589e363cd36541abda1d60fe1cf6f4ae9b00 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 22 Aug 2024 15:44:40 -0500 Subject: [PATCH 22/34] forgot another keyword surmise assignment --- libensemble/tests/regression_tests/test_asktell_surmise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_asktell_surmise.py b/libensemble/tests/regression_tests/test_asktell_surmise.py index a4e5d9ae9..d0aa5310c 100644 --- a/libensemble/tests/regression_tests/test_asktell_surmise.py +++ b/libensemble/tests/regression_tests/test_asktell_surmise.py @@ -80,7 +80,7 @@ } persis_info = add_unique_random_streams({}, 5) - surmise = Surmise(gen_specs, persis_info=persis_info[1]) # we add sim_id as a field to gen_specs["out"] + surmise = Surmise(gen_specs=gen_specs, persis_info=persis_info[1]) # we add sim_id as a field to gen_specs["out"] surmise.setup() initial_sample = surmise.ask() From 60dd2035152dd6ec337ddc36bd3046130ddaad4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:30:56 +0000 Subject: [PATCH 23/34] Bump crate-ci/typos from 1.23.7 to 1.24.1 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.23.7 to 1.24.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/v1.23.7...v1.24.1) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/basic.yml | 2 +- .github/workflows/extra.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index ef6f92eb9..ffa2b2d50 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -163,4 +163,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.23.7 + - uses: crate-ci/typos@v1.24.1 diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index f493c08f2..1af0cb737 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -275,4 +275,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.23.7 + - uses: crate-ci/typos@v1.24.1 From 5f33724ecf6ae5f2a86d39e48dd4f61d0cafaa32 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2024 11:36:30 -0500 Subject: [PATCH 24/34] add unit test for awkward H and checking routine from shuds, add case for np_to_list_dicts to unpack length-1 arrays/lists, into scalars --- libensemble/tests/unit_tests/test_asktell.py | 38 ++++++++++++++++++-- libensemble/utils/misc.py | 5 ++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index dbdc4148d..ed25ac7bb 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -4,6 +4,24 @@ from libensemble.utils.misc import list_dicts_to_np +def _check_conversion(H, npp): + + for field in H.dtype.names: + print(f"Comparing {field}: {H[field]} {npp[field]}") + + if isinstance(H[field], np.ndarray): + assert np.array_equal(H[field], npp[field]), f"Mismatch found in field {field}" + + elif isinstance(H[field], str) and isinstance(npp[field], str): + assert H[field] == npp[field], f"Mismatch found in field {field}" + + elif np.isscalar(H[field]) and np.isscalar(npp[field]): + assert np.isclose(H[field], npp[field]), f"Mismatch found in field {field}" + + else: + raise TypeError(f"Unhandled or mismatched types in field {field}: {type(H[field])} vs {type(npp[field])}") + + def test_asktell_sampling_and_utils(): from libensemble.gen_classes.sampling import UniformSample @@ -38,10 +56,12 @@ def test_asktell_sampling_and_utils(): assert value == out_np["x"][i][j] -def test_additional_converts(): +def test_awkward_list_dict(): from libensemble.utils.misc import list_dicts_to_np # test list_dicts_to_np on a weirdly formatted dictionary + # Unfortunately, we're not really checking against some original + # libE-styled source of truth, like H. out_np = list_dicts_to_np( [ { @@ -67,6 +87,20 @@ def test_additional_converts(): assert all([i in ("x", "y", "z", "a0") for i in out_np.dtype.names]) +def test_awkward_H(): + from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts + + dtype = [("a", "i4"), ("x", "f4", (3,)), ("y", "f4", (1,)), ("z", "f4", (12,)), ("greeting", "U10"), ("co2", "f8")] + H = np.zeros(2, dtype=dtype) + H[0] = (1, [1.1, 2.2, 3.3], [10.1], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "hello", "1.23") + H[1] = (2, [4.4, 5.5, 6.6], [11.1], [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "goodbye", "2.23") + + list_dicts = np_to_list_dicts(H) + npp = list_dicts_to_np(list_dicts) + _check_conversion(H, npp) + + if __name__ == "__main__": test_asktell_sampling_and_utils() - test_additional_converts() + test_awkward_list_dict() + test_awkward_H() diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index 659a97440..1e03beab6 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -150,9 +150,12 @@ def np_to_list_dicts(array: npt.NDArray) -> List[dict]: for row in array: new_dict = {} for field in row.dtype.names: - if hasattr(row[field], "__len__") and len(row[field]) > 1: + # non-string arrays, lists, etc. + if hasattr(row[field], "__len__") and len(row[field]) > 1 and not isinstance(row[field], str): for i, x in enumerate(row[field]): new_dict[field + str(i)] = x + elif hasattr(row[field], "__len__") and len(row[field]) == 1: # single-entry arrays, lists, etc. + new_dict[field] = row[field][0] # will still work on single-char strings else: new_dict[field] = row[field] out.append(new_dict) From 41c16b7c79d34ecd159f36c17da7da23255b6dee Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2024 12:17:48 -0500 Subject: [PATCH 25/34] add optional dtype argument for list_dicts_to_np to preempt "dtype discovery" routine. formatting --- libensemble/tests/unit_tests/test_asktell.py | 45 ++++++++++---------- libensemble/utils/misc.py | 18 ++++---- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index ed25ac7bb..9e60550e8 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -62,27 +62,28 @@ def test_awkward_list_dict(): # test list_dicts_to_np on a weirdly formatted dictionary # Unfortunately, we're not really checking against some original # libE-styled source of truth, like H. - out_np = list_dicts_to_np( - [ - { - "x0": "abcd", - "x1": "efgh", - "y": 56, - "z0": 1, - "z1": 2, - "z2": 3, - "z3": 4, - "z4": 5, - "z5": 6, - "z6": 7, - "z7": 8, - "z8": 9, - "z9": 10, - "z10": 11, - "a0": "B", - } - ] - ) + + weird_list_dict = [ + { + "x0": "abcd", + "x1": "efgh", + "y": 56, + "z0": 1, + "z1": 2, + "z2": 3, + "z3": 4, + "z4": 5, + "z5": 6, + "z6": 7, + "z7": 8, + "z8": 9, + "z9": 10, + "z10": 11, + "a0": "B", + } + ] + + out_np = list_dicts_to_np(weird_list_dict) assert all([i in ("x", "y", "z", "a0") for i in out_np.dtype.names]) @@ -96,7 +97,7 @@ def test_awkward_H(): H[1] = (2, [4.4, 5.5, 6.6], [11.1], [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "goodbye", "2.23") list_dicts = np_to_list_dicts(H) - npp = list_dicts_to_np(list_dicts) + npp = list_dicts_to_np(list_dicts, dtype=dtype) _check_conversion(H, npp) diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index 1e03beab6..d242edf65 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -108,7 +108,7 @@ def _combine_names(names: list) -> list: return list(set(out_names)) -def list_dicts_to_np(list_dicts: list) -> npt.NDArray: +def list_dicts_to_np(list_dicts: list, dtype: list = None) -> npt.NDArray: if list_dicts is None: return None @@ -122,15 +122,17 @@ def list_dicts_to_np(list_dicts: list) -> npt.NDArray: else: # single name, e.g. local_pt, a0 *AS LONG AS THERE ISNT AN A1* combinable_names.append([name]) - new_dtype = [] + if dtype is None: + dtype = [] - # another loop over names, there's probably a more elegant way, but my brain is fried - for i, entry in enumerate(combinable_names): - name = new_dtype_names[i] - size = len(combinable_names[i]) - new_dtype.append(_decide_dtype(name, first[entry[0]], size)) + if not len(dtype): + # another loop over names, there's probably a more elegant way, but my brain is fried + for i, entry in enumerate(combinable_names): + name = new_dtype_names[i] + size = len(combinable_names[i]) + dtype.append(_decide_dtype(name, first[entry[0]], size)) - out = np.zeros(len(list_dicts), dtype=new_dtype) + out = np.zeros(len(list_dicts), dtype=dtype) for i, group in enumerate(combinable_names): new_dtype_name = new_dtype_names[i] From 48604287b99c207a9c8dca012598abf3d9ec2a80 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2024 12:51:23 -0500 Subject: [PATCH 26/34] replace _to_array with list_dicts_to_np with dtype parameter. list_dicts_to_np passes through input as-is if its not a list (already numpy, no conversion necessary. _to_array did this previously) --- libensemble/utils/misc.py | 3 +++ libensemble/utils/runners.py | 21 ++++----------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index d242edf65..34b7a0931 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -112,6 +112,9 @@ def list_dicts_to_np(list_dicts: list, dtype: list = None) -> npt.NDArray: if list_dicts is None: return None + if not isinstance(list_dicts, list): # presumably already a numpy array, conversion not necessary + return list_dicts + first = list_dicts[0] # for determining dtype of output np array new_dtype_names = _combine_names([i for i in first.keys()]) # -> ['x', 'y'] combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2'], ['z']] diff --git a/libensemble/utils/runners.py b/libensemble/utils/runners.py index d688a427e..fe9a9fa2a 100644 --- a/libensemble/utils/runners.py +++ b/libensemble/utils/runners.py @@ -4,14 +4,13 @@ import time from typing import Optional -import numpy as np import numpy.typing as npt from libensemble.comms.comms import QCommThread from libensemble.generators import LibensembleGenerator, LibensembleGenThreadInterfacer from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG from libensemble.tools.persistent_support import PersistentSupport -from libensemble.utils.misc import np_to_list_dicts +from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts logger = logging.getLogger(__name__) @@ -107,22 +106,9 @@ def __init__(self, specs): super().__init__(specs) self.gen = specs.get("generator") - def _to_array(self, x: list) -> npt.NDArray: - """fast-cast list-of-dicts to NumPy array""" - if isinstance(x, list) and len(x) and isinstance(x[0], dict): - arr = np.zeros(len(x), dtype=self.specs["out"]) - for i in range(len(x)): - for key in x[0].keys(): - arr[i][key] = x[i][key] - return arr - return x - def _get_points_updates(self, batch_size: int) -> (npt.NDArray, npt.NDArray): # no ask_updates on external gens - return ( - self._to_array(self.gen.ask(batch_size)), - None, - ) + return (list_dicts_to_np(self.gen.ask(batch_size), dtype=self.gen_specs["out"]), None) def _convert_tell(self, x: npt.NDArray) -> list: self.gen.tell(np_to_list_dicts(x)) @@ -155,7 +141,8 @@ def _persistent_result(self, calc_in, persis_info, libE_info): self.gen.libE_info = libE_info if self.gen.thread is None: self.gen.setup() # maybe we're reusing a live gen from a previous run - H_out = self._to_array(self._get_initial_ask(libE_info)) + # libE gens will hit the following line, but list_dicts_to_np will passthrough if the output is a numpy array + H_out = list_dicts_to_np(self._get_initial_ask(libE_info), dtype=self.specs["out"]) tag, Work, H_in = self.ps.send_recv(H_out) # evaluate the initial sample final_H_in = self._start_generator_loop(tag, Work, H_in) return self.gen.final_tell(final_H_in), FINISHED_PERSISTENT_GEN_TAG From ced8992b3bd8bbd93d8351a1c5fad7f0e1918911 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2024 13:10:59 -0500 Subject: [PATCH 27/34] fix --- libensemble/utils/runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/utils/runners.py b/libensemble/utils/runners.py index fe9a9fa2a..1d94fa097 100644 --- a/libensemble/utils/runners.py +++ b/libensemble/utils/runners.py @@ -108,7 +108,7 @@ def __init__(self, specs): def _get_points_updates(self, batch_size: int) -> (npt.NDArray, npt.NDArray): # no ask_updates on external gens - return (list_dicts_to_np(self.gen.ask(batch_size), dtype=self.gen_specs["out"]), None) + return (list_dicts_to_np(self.gen.ask(batch_size), dtype=self.specs["out"]), None) def _convert_tell(self, x: npt.NDArray) -> list: self.gen.tell(np_to_list_dicts(x)) From 5f6654b89c7114fec108623326ca7cb380443ac9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 28 Aug 2024 11:32:14 -0500 Subject: [PATCH 28/34] better surmise link, no longer point to fork --- docs/examples/surmise.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/examples/surmise.rst b/docs/examples/surmise.rst index 72fa87413..69d0f068e 100644 --- a/docs/examples/surmise.rst +++ b/docs/examples/surmise.rst @@ -2,15 +2,11 @@ persistent_surmise ------------------ Required: Surmise_ - -Note that currently the github fork https://github.com/mosesyhc/surmise should be used:: - - pip install --upgrade git+https://github.com/bandframework/surmise.git@develop - -The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an example of the capability to cancel pending simulations. +The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an +example of the capability to cancel pending simulations. .. automodule:: persistent_surmise_calib :members: :no-undoc-members: -.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html +.. _Surmise: https://github.com/bandframework/surmise From 4261ca889ad99d5c1aaa723f81ec9d62ecaef4ed Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 29 Aug 2024 14:33:05 -0500 Subject: [PATCH 29/34] LibensembleGenerator can provide matching dtype for list_dicts_to_np, but its only necessary within the ask() --- libensemble/generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/generators.py b/libensemble/generators.py index 5e9d957b4..9fa450123 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -117,7 +117,7 @@ def ask(self, num_points: Optional[int] = 0) -> List[dict]: def tell(self, results: List[dict]) -> None: """Send the results of evaluations to the generator.""" - self.tell_numpy(list_dicts_to_np(results)) + self.tell_numpy(list_dicts_to_np(results), dtype=self.gen_specs.get("out")) class LibensembleGenThreadInterfacer(LibensembleGenerator): From 460bbe346dc0f9530275b3a3a47f3b88a318853c Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 29 Aug 2024 15:51:52 -0500 Subject: [PATCH 30/34] fix --- libensemble/generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/generators.py b/libensemble/generators.py index 9fa450123..74c8682e1 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -117,7 +117,7 @@ def ask(self, num_points: Optional[int] = 0) -> List[dict]: def tell(self, results: List[dict]) -> None: """Send the results of evaluations to the generator.""" - self.tell_numpy(list_dicts_to_np(results), dtype=self.gen_specs.get("out")) + self.tell_numpy(list_dicts_to_np(results, dtype=self.gen_specs.get("out"))) class LibensembleGenThreadInterfacer(LibensembleGenerator): From 7fdd8a662845900636c9390b4c0040f7092e3e64 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 29 Aug 2024 15:55:25 -0500 Subject: [PATCH 31/34] ahhhh, just gen_specs['out']'s dtype isn't sufficient. persis_in, describing the names of the fields, decides what fields are passed in, but their "actual datatypes" come from the sim / sim_specs --- libensemble/generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/generators.py b/libensemble/generators.py index 74c8682e1..70eac32e1 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -117,7 +117,7 @@ def ask(self, num_points: Optional[int] = 0) -> List[dict]: def tell(self, results: List[dict]) -> None: """Send the results of evaluations to the generator.""" - self.tell_numpy(list_dicts_to_np(results, dtype=self.gen_specs.get("out"))) + self.tell_numpy(list_dicts_to_np(results)) # OH, we need the union of sim_specs.out and gen_specs.out class LibensembleGenThreadInterfacer(LibensembleGenerator): From 69b0584cc9282ca28cbb147a4a8f3e9912f6029f Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 12 Sep 2024 12:33:04 -0500 Subject: [PATCH 32/34] removing hardcoded gen_specs.out, removing hardcoded persis_info.nworkers, use gen_specs.get("out") so if it isnt provided, the dtype discovery process commences --- libensemble/gen_classes/aposmm.py | 3 +-- libensemble/gen_classes/sampling.py | 2 -- libensemble/generators.py | 1 - libensemble/utils/runners.py | 4 ++-- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/libensemble/gen_classes/aposmm.py b/libensemble/gen_classes/aposmm.py index d49832730..108282e07 100644 --- a/libensemble/gen_classes/aposmm.py +++ b/libensemble/gen_classes/aposmm.py @@ -30,8 +30,7 @@ def __init__( ] gen_specs["persis_in"] = ["x", "f", "local_pt", "sim_id", "sim_ended", "x_on_cube", "local_min"] if not persis_info: - persis_info = add_unique_random_streams({}, 4, seed=4321)[1] - persis_info["nworkers"] = 4 + persis_info = add_unique_random_streams({}, 2, seed=4321)[1] super().__init__(History, persis_info, gen_specs, libE_info, **kwargs) self.all_local_minima = [] self.results_idx = 0 diff --git a/libensemble/gen_classes/sampling.py b/libensemble/gen_classes/sampling.py index dd347db51..166286482 100644 --- a/libensemble/gen_classes/sampling.py +++ b/libensemble/gen_classes/sampling.py @@ -34,7 +34,6 @@ class UniformSample(SampleBase): def __init__(self, _=[], persis_info={}, gen_specs={}, libE_info=None, **kwargs): super().__init__(_, persis_info, gen_specs, libE_info, **kwargs) self._get_user_params(self.gen_specs["user"]) - self.gen_specs["out"] = [("x", float, (self.n,))] def ask_numpy(self, n_trials): H_o = np.zeros(n_trials, dtype=self.gen_specs["out"]) @@ -60,7 +59,6 @@ def __init__(self, _, persis_info, gen_specs, libE_info=None, **kwargs): self.gen_specs = gen_specs self.persis_info = persis_info self._get_user_params(self.gen_specs["user"]) - self.gen_specs["out"] = [("x", float, (self.n,))] def ask(self, n_trials): H_o = [] diff --git a/libensemble/generators.py b/libensemble/generators.py index 70eac32e1..37b974139 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -99,7 +99,6 @@ def __init__( self.gen_specs["user"] = kwargs if not persis_info: self.persis_info = add_unique_random_streams({}, 4, seed=4321)[1] - self.persis_info["nworkers"] = 4 else: self.persis_info = persis_info diff --git a/libensemble/utils/runners.py b/libensemble/utils/runners.py index 1d94fa097..08d52a27e 100644 --- a/libensemble/utils/runners.py +++ b/libensemble/utils/runners.py @@ -108,7 +108,7 @@ def __init__(self, specs): def _get_points_updates(self, batch_size: int) -> (npt.NDArray, npt.NDArray): # no ask_updates on external gens - return (list_dicts_to_np(self.gen.ask(batch_size), dtype=self.specs["out"]), None) + return (list_dicts_to_np(self.gen.ask(batch_size), dtype=self.specs.get("out")), None) def _convert_tell(self, x: npt.NDArray) -> list: self.gen.tell(np_to_list_dicts(x)) @@ -142,7 +142,7 @@ def _persistent_result(self, calc_in, persis_info, libE_info): if self.gen.thread is None: self.gen.setup() # maybe we're reusing a live gen from a previous run # libE gens will hit the following line, but list_dicts_to_np will passthrough if the output is a numpy array - H_out = list_dicts_to_np(self._get_initial_ask(libE_info), dtype=self.specs["out"]) + H_out = list_dicts_to_np(self._get_initial_ask(libE_info), dtype=self.specs.get("out")) tag, Work, H_in = self.ps.send_recv(H_out) # evaluate the initial sample final_H_in = self._start_generator_loop(tag, Work, H_in) return self.gen.final_tell(final_H_in), FINISHED_PERSISTENT_GEN_TAG From 8c01ca95f76d1f9d1edb3c59333bcdb0c92c448d Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 12 Sep 2024 12:35:59 -0500 Subject: [PATCH 33/34] clarify a comment --- libensemble/generators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libensemble/generators.py b/libensemble/generators.py index 37b974139..b13bae31c 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -116,7 +116,9 @@ def ask(self, num_points: Optional[int] = 0) -> List[dict]: def tell(self, results: List[dict]) -> None: """Send the results of evaluations to the generator.""" - self.tell_numpy(list_dicts_to_np(results)) # OH, we need the union of sim_specs.out and gen_specs.out + self.tell_numpy(list_dicts_to_np(results)) + # Note that although we'd prefer to have a complete dtype available, the gen + # doesn't have access to sim_specs["out"] currently. class LibensembleGenThreadInterfacer(LibensembleGenerator): From 4541d8afbdf45b3132fa881035b91ad6d7a200d2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 12 Sep 2024 14:15:44 -0500 Subject: [PATCH 34/34] as discussed, currently gen_specs['out'] must be provided to a gen instead of it deciding it for itself internally --- libensemble/tests/unit_tests/test_asktell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index 9e60550e8..fd80b8829 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -39,7 +39,7 @@ def test_asktell_sampling_and_utils(): assert len(gen.ask(10)) == 10 # Test initialization gen-specific keyword args - gen = UniformSample(lb=np.array([-3, -2]), ub=np.array([3, 2])) + gen = UniformSample(gen_specs=gen_specs, lb=np.array([-3, -2]), ub=np.array([3, 2])) assert len(gen.ask(10)) == 10 out_np = gen.ask_numpy(3) # should get numpy arrays, non-flattened