From 56a70a7fe2d1caedee3806a5fd0cbdcb665eab7b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 30 Aug 2023 13:40:43 -0500 Subject: [PATCH 01/10] tentatively-named decorators for setting user-function data attributes. SimSpecs and GenSpecs can extract and set --- libensemble/sim_funcs/one_d_func.py | 4 ++ libensemble/specs.py | 50 ++++++++++++++++--- .../regression_tests/test_1d_sampling.py | 2 +- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/libensemble/sim_funcs/one_d_func.py b/libensemble/sim_funcs/one_d_func.py index ea87e2d0c..20fbec8d5 100644 --- a/libensemble/sim_funcs/one_d_func.py +++ b/libensemble/sim_funcs/one_d_func.py @@ -5,7 +5,11 @@ import numpy as np +from libensemble.specs import input_fields, output_data + +@input_fields(["x"]) +@output_data([("f", float)]) def one_d_example(x, persis_info, sim_specs, _): """ Evaluates the six hump camel function for a single point ``x``. diff --git a/libensemble/specs.py b/libensemble/specs.py index df9161c5a..fa4a97815 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -7,9 +7,7 @@ from pydantic import BaseConfig, BaseModel, Field, root_validator, validator from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.gen_funcs.sampling import latin_hypercube_sample from libensemble.resources.platforms import Platform -from libensemble.sim_funcs.one_d_func import one_d_example from libensemble.utils.specs_checkers import ( _check_any_workers_and_disable_rm_if_tcp, _check_exit_criteria, @@ -40,13 +38,13 @@ class SimSpecs(BaseModel): a ``sim_specs`` dictionary. """ - sim_f: Callable = one_d_example + sim_f: Callable = None """ Python function that matches the ``sim_f`` api. e.g. ``libensemble.sim_funcs.borehole``. Evaluates parameters produced by a generator function """ - inputs: List[str] = Field([], alias="in") + inputs: Optional[List[str]] = Field([], alias="in") """ List of field names out of the complete history to pass into the simulation function on initialization. Can use ``in`` or ``inputs`` as keyword. @@ -59,7 +57,7 @@ class SimSpecs(BaseModel): """ # list of tuples for dtype construction - out: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]] = [] + out: Optional[List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]] = [] """ List of tuples corresponding to NumPy dtypes. e.g. ``("dim", int, (3,))``, or ``("path", str)``. Typically used to initialize an output array within the simulation function: @@ -95,6 +93,18 @@ def check_valid_in(cls, v): raise ValueError(_IN_INVALID_ERR) return v + @root_validator + def set_in_out_from_attrs(cls, values): + if not values.get("sim_f"): + from libensemble.sim_funcs.one_d_func import one_d_example + + values["sim_f"] = one_d_example + if hasattr(values.get("sim_f"), "inputs") and not values.get("inputs"): + values["inputs"] = values.get("sim_f").inputs + if hasattr(values.get("sim_f"), "outputs") and not values.get("out"): + values["out"] = values.get("sim_f").outputs + return values + class GenSpecs(BaseModel): """ @@ -102,7 +112,7 @@ class GenSpecs(BaseModel): a ``gen_specs`` dictionary. """ - gen_f: Optional[Callable] = latin_hypercube_sample + gen_f: Optional[Callable] = None """ Python function that matches the gen_f api. e.g. `libensemble.gen_funcs.sampling`. Produces parameters for evaluation by a simulator function, and makes decisions based on simulator function output @@ -156,6 +166,18 @@ def check_valid_in(cls, v): raise ValueError(_IN_INVALID_ERR) return v + @root_validator + def set_in_out_from_attrs(cls, values): + if not values.get("gen_f"): + from libensemble.gen_funcs.sampling import latin_hypercube_sample + + values["gen_f"] = latin_hypercube_sample + if hasattr(values.get("gen_f"), "inputs") and not values.get("inputs"): + values["inputs"] = values.get("gen_f").inputs + if hasattr(values.get("gen_f"), "outputs") and not values.get("out"): + values["out"] = values.get("gen_f").outputs + return values + class AllocSpecs(BaseModel): """ @@ -576,3 +598,19 @@ def check_H0(cls, values): if values.get("H0") is not None: return _check_H0(values) return values + + +def input_fields(fields: List[str]): + def decorator(func): + setattr(func, "inputs", fields) + return func + + return decorator + + +def output_data(fields: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]): + def decorator(func): + setattr(func, "outputs", fields) + return func + + return decorator diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index 6a21b8e27..3eafea32a 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -27,7 +27,7 @@ sampling = Ensemble(parse_args=True) sampling.libE_specs = LibeSpecs(save_every_k_gens=300, safe_mode=False, disable_log_files=True) - sampling.sim_specs = SimSpecs(sim_f=sim_f, inputs=["x"], out=[("f", float)]) + sampling.sim_specs = SimSpecs(sim_f=sim_f) sampling.gen_specs = GenSpecs( gen_f=gen_f, out=[("x", float, (1,))], From 833c96c2595660d0eda2c5038b98675dfa7421aa Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 5 Sep 2023 13:50:56 -0500 Subject: [PATCH 02/10] wrappers append input/output attributes to top of wrapped function docstrings --- libensemble/sim_funcs/six_hump_camel.py | 3 +++ libensemble/specs.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index 415af0050..638887507 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -17,9 +17,12 @@ import numpy as np from libensemble.message_numbers import EVAL_SIM_TAG, FINISHED_PERSISTENT_SIM_TAG, PERSIS_STOP, STOP_TAG +from libensemble.specs import input_fields, output_data from libensemble.tools.persistent_support import PersistentSupport +@input_fields(["x"]) +@output_data([("f", float)]) def six_hump_camel(H, persis_info, sim_specs, libE_info): """ Evaluates the six hump camel function for a collection of points given in ``H["x"]``. diff --git a/libensemble/specs.py b/libensemble/specs.py index 21bb4bf48..53958566d 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -617,6 +617,7 @@ def check_H0(cls, values): def input_fields(fields: List[str]): def decorator(func): setattr(func, "inputs", fields) + func.__doc__ = f"\n Input Fields: ``{func.inputs}``\n" + func.__doc__ return func return decorator @@ -625,6 +626,7 @@ def decorator(func): def output_data(fields: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]): def decorator(func): setattr(func, "outputs", fields) + func.__doc__ = f"\n Output Datatypes: ``{func.outputs}``\n" + func.__doc__ return func return decorator From 9136746402f7b6d161d63fd2ce4a0608fadc626b Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 5 Sep 2023 14:41:09 -0500 Subject: [PATCH 03/10] bold labels --- libensemble/specs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/specs.py b/libensemble/specs.py index 53958566d..d9b2443a9 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -617,7 +617,7 @@ def check_H0(cls, values): def input_fields(fields: List[str]): def decorator(func): setattr(func, "inputs", fields) - func.__doc__ = f"\n Input Fields: ``{func.inputs}``\n" + func.__doc__ + func.__doc__ = f"\n **Input Fields:** ``{func.inputs}``\n" + func.__doc__ return func return decorator @@ -626,7 +626,7 @@ def decorator(func): def output_data(fields: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]): def decorator(func): setattr(func, "outputs", fields) - func.__doc__ = f"\n Output Datatypes: ``{func.outputs}``\n" + func.__doc__ + func.__doc__ = f"\n **Output Datatypes:** ``{func.outputs}``\n" + func.__doc__ return func return decorator From b314e152797c105ad4fe31bc679a5bcd99ef2f2d Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 7 Sep 2023 15:45:18 -0500 Subject: [PATCH 04/10] add persistent_input_fields decorator --- libensemble/specs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/libensemble/specs.py b/libensemble/specs.py index 6b72f3a63..31084b35a 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -101,8 +101,10 @@ def set_in_out_from_attrs(cls, values): values["sim_f"] = one_d_example if hasattr(values.get("sim_f"), "inputs") and not values.get("inputs"): values["inputs"] = values.get("sim_f").inputs - if hasattr(values.get("sim_f"), "outputs") and not values.get("out"): + if hasattr(values.get("sim_f"), "outputs") and not values.get("outputs"): values["out"] = values.get("sim_f").outputs + if hasattr(values.get("sim_f"), "persis_in") and not values.get("persis_in"): + values["persis_in"] = values.get("sim_f").persis_in return values @@ -174,8 +176,10 @@ def set_in_out_from_attrs(cls, values): values["gen_f"] = latin_hypercube_sample if hasattr(values.get("gen_f"), "inputs") and not values.get("inputs"): values["inputs"] = values.get("gen_f").inputs - if hasattr(values.get("gen_f"), "outputs") and not values.get("out"): + if hasattr(values.get("gen_f"), "outputs") and not values.get("outputs"): values["out"] = values.get("gen_f").outputs + if hasattr(values.get("gen_f"), "persis_in") and not values.get("persis_in"): + values["persis_in"] = values.get("gen_f").persis_in return values @@ -623,6 +627,15 @@ def decorator(func): return decorator +def persistent_input_fields(fields: List[str]): + def decorator(func): + setattr(func, "persis_in", fields) + func.__doc__ = f"\n **Persistent Input Fields:** ``{func.persis_inputs}``\n" + func.__doc__ + return func + + return decorator + + def output_data(fields: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]): def decorator(func): setattr(func, "outputs", fields) From 1c3212456be767fe3b789a345512c00c12210fcf Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 8 Sep 2023 15:06:38 -0500 Subject: [PATCH 05/10] initial docstrings for decorator functions --- libensemble/specs.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/libensemble/specs.py b/libensemble/specs.py index 31084b35a..63e4947d4 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -619,6 +619,23 @@ def check_H0(cls, values): def input_fields(fields: List[str]): + """Decorates a user-function with a list of field names to pass in on initialization. + + Decorated functions don't need those fields specified in ``SimSpecs.inputs`` or ``GenSpecs.inputs``. + + .. code-block:: python + + from libensemble.specs import input_fields, output_data + + + @input_fields(["x"]) + @output_data([("f", float)]) + def one_d_example(x, persis_info, sim_specs): + H_o = np.zeros(1, dtype=sim_specs["out"]) + H_o["f"] = np.linalg.norm(x) + return H_o, persis_info + """ + def decorator(func): setattr(func, "inputs", fields) func.__doc__ = f"\n **Input Fields:** ``{func.inputs}``\n" + func.__doc__ @@ -628,6 +645,33 @@ def decorator(func): def persistent_input_fields(fields: List[str]): + """Decorates a *persistent* user-function with a list of field names to send in throughout runtime. + + Decorated functions don't need those fields specified in ``SimSpecs.persis_in`` or ``GenSpecs.persis_in``. + + .. code-block:: python + + from libensemble.specs import persistent_input_fields, output_data + + + @persistent_input_fields(["f"]) + @output_data(["x", float]) + def persistent_uniform(_, persis_info, gen_specs, libE_info): + + b, n, lb, ub = _get_user_params(gen_specs["user"]) + ps = PersistentSupport(libE_info, EVAL_GEN_TAG) + + tag = None + while tag not in [STOP_TAG, PERSIS_STOP]: + H_o = np.zeros(b, dtype=gen_specs["out"]) + H_o["x"] = persis_info["rand_stream"].uniform(lb, ub, (b, n)) + tag, Work, calc_in = ps.send_recv(H_o) + if hasattr(calc_in, "__len__"): + b = len(calc_in) + + return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + """ + def decorator(func): setattr(func, "persis_in", fields) func.__doc__ = f"\n **Persistent Input Fields:** ``{func.persis_inputs}``\n" + func.__doc__ @@ -637,6 +681,23 @@ def decorator(func): def output_data(fields: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]): + """Decorates a user-function with a list of tuples corresponding to NumPy dtypes for the function's output data. + + Decorated functions don't need those fields specified in ``SimSpecs.outputs`` or ``GenSpecs.outputs``. + + .. code-block:: python + + from libensemble.specs import input_fields, output_data + + + @input_fields(["x"]) + @output_data([("f", float)]) + def one_d_example(x, persis_info, sim_specs): + H_o = np.zeros(1, dtype=sim_specs["out"]) + H_o["f"] = np.linalg.norm(x) + return H_o, persis_info + """ + def decorator(func): setattr(func, "outputs", fields) func.__doc__ = f"\n **Output Datatypes:** ``{func.outputs}``\n" + func.__doc__ From efa1dbb050df21c0a9363a9425d5508ccbb30935 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 9 Oct 2023 11:29:49 -0500 Subject: [PATCH 06/10] first pass on briefly mentioning decorators in generator function guide --- docs/function_guides/generator.rst | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/function_guides/generator.rst b/docs/function_guides/generator.rst index c8d0b3ef6..c8499b4a3 100644 --- a/docs/function_guides/generator.rst +++ b/docs/function_guides/generator.rst @@ -5,8 +5,13 @@ Generator Functions Generator and :ref:`Simulator functions` have relatively similar interfaces. +Writing a Generator +------------------- + .. code-block:: python + @input_fields(["f"]) + @output_data([("x", float)]) def my_generator(Input, persis_info, gen_specs, libE_info): batch_size = gen_specs["user"]["batch_size"] @@ -22,19 +27,23 @@ Most ``gen_f`` function definitions written by users resemble:: where: - * ``Input`` is a selection of the :ref:`History array` - * :ref:`persis_info` is a dictionary containing state information - * :ref:`gen_specs` is a dictionary of generator parameters, including which fields from the History array got sent - * ``libE_info`` is a dictionary containing libEnsemble-specific entries + * ``Input`` is a selection of the :ref:`History array`, a NumPy array. + * :ref:`persis_info` is a dictionary containing state information. + * :ref:`gen_specs` is a dictionary of generator parameters. + * ``libE_info`` is a dictionary containing miscellaneous entries. + +*Optional* ``input_fields`` and ``output_data`` decorators for the function describe the +fields to pass in and the output data format. Otherwise those fields +need to be specified in :class:`GenSpecs`. Valid generator functions can accept a subset of the above parameters. So a very simple generator can start:: def my_generator(Input): -If gen_specs was initially defined:: +If ``gen_specs`` was initially defined:: gen_specs = { - "gen_f": some_function, + "gen_f": my_generator, "in": ["f"], "out:" ["x", float, (1,)], "user": { From 4e7217a09848bb686d681e583ea9334c238e26d2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 9 Oct 2023 13:13:37 -0500 Subject: [PATCH 07/10] use synced tabs to compare decorated and non-decorated functions and their specs --- docs/data_structures/persis_info.rst | 2 +- docs/function_guides/generator.rst | 91 +++++++++++++++++----------- docs/function_guides/simulator.rst | 82 +++++++++++++++++-------- 3 files changed, 114 insertions(+), 61 deletions(-) diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 841090298..7bdc59c90 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -55,7 +55,7 @@ Examples: .. literalinclude:: ../../libensemble/alloc_funcs/start_only_persistent.py :linenos: :start-at: if gen_count < persis_info.get("num_gens_started", 0): - :end-before: # Give evaluated results back to the persistent gen + :end-before: # Give evaluated results back to a running persistent gen :emphasize-lines: 1 :caption: libensemble/alloc_funcs/start_only_persistent.py diff --git a/docs/function_guides/generator.rst b/docs/function_guides/generator.rst index c8499b4a3..7f7a7bcac 100644 --- a/docs/function_guides/generator.rst +++ b/docs/function_guides/generator.rst @@ -8,18 +8,37 @@ Generator and :ref:`Simulator functions` have relatively similar Writing a Generator ------------------- -.. code-block:: python +.. tab-set:: - @input_fields(["f"]) - @output_data([("x", float)]) - def my_generator(Input, persis_info, gen_specs, libE_info): - batch_size = gen_specs["user"]["batch_size"] + .. tab-item:: Non-decorated + :sync: nodecorate + + .. code-block:: python + + def my_generator(Input, persis_info, gen_specs, libE_info): + batch_size = gen_specs["user"]["batch_size"] + + Output = np.zeros(batch_size, gen_specs["out"]) + # ... + Output["x"], persis_info = generate_next_simulation_inputs(Input["f"], persis_info) + + return Output, persis_info + + .. tab-item:: Decorated + :sync: decorate + + .. code-block:: python - Output = np.zeros(batch_size, gen_specs["out"]) - ... - Output["x"], persis_info = generate_next_simulation_inputs(Input["f"], persis_info) + @input_fields(["f"]) + @output_data([("x", float)]) + def my_generator(Input, persis_info, gen_specs, libE_info): + batch_size = gen_specs["user"]["batch_size"] - return Output, persis_info + Output = np.zeros(batch_size, gen_specs["out"]) + # ... + Output["x"], persis_info = generate_next_simulation_inputs(Input["f"], persis_info) + + return Output, persis_info Most ``gen_f`` function definitions written by users resemble:: @@ -32,24 +51,27 @@ where: * :ref:`gen_specs` is a dictionary of generator parameters. * ``libE_info`` is a dictionary containing miscellaneous entries. -*Optional* ``input_fields`` and ``output_data`` decorators for the function describe the -fields to pass in and the output data format. Otherwise those fields -need to be specified in :class:`GenSpecs`. - Valid generator functions can accept a subset of the above parameters. So a very simple generator can start:: def my_generator(Input): -If ``gen_specs`` was initially defined:: +If ``gen_specs`` was initially defined: + +.. tab-set:: + + .. tab-item:: Non-decorated function + :sync: nodecorate + + .. code-block:: python + + gen_specs = GenSpecs(gen_f=my_generator, inputs=["f"], outputs=["x", float, (1,)], user={"batch_size": 128}) + + .. tab-item:: Decorated function + :sync: decorate + + .. code-block:: python - gen_specs = { - "gen_f": my_generator, - "in": ["f"], - "out:" ["x", float, (1,)], - "user": { - "batch_size": 128 - } - } + gen_specs = GenSpecs(gen_f=my_generator, user={"batch_size": 128}) Then user parameters and a *local* array of outputs may be obtained/initialized like:: @@ -65,10 +87,9 @@ Then return the array and ``persis_info`` to libEnsemble:: return Output, persis_info -Between the ``Output`` definition and the ``return``, any level and complexity -of computation can be performed. Users are encouraged to use the :doc:`executor<../executor/overview>` -to submit applications to parallel resources if necessary, or plug in components from -other libraries to serve their needs. +Between the ``Output`` definition and the ``return``, any computation can be performed. +Users can try an :doc:`executor<../executor/overview>` to submit applications to parallel +resources, or plug in components from other libraries to serve their needs. .. note:: @@ -83,17 +104,17 @@ Persistent Generators While non-persistent generators return after completing their calculation, persistent generators do the following in a loop: - 1. Receive simulation results and metadata; exit if metadata instructs - 2. Perform analysis - 3. Send subsequent simulation parameters + 1. Receive simulation results and metadata; exit if metadata instructs. + 2. Perform analysis. + 3. Send subsequent simulation parameters. Persistent generators don't need to be re-initialized on each call, but are typically -more complicated. The :doc:`APOSMM<../examples/aposmm>` -optimization generator function included with libEnsemble is persistent so it can -maintain multiple local optimization subprocesses based on results from complete simulations. +more complicated. The persistent :doc:`APOSMM<../examples/aposmm>` +optimization generator function included with libEnsemble maintains +local optimization subprocesses based on results from complete simulations. -Use ``gen_specs["persis_in"]`` to specify fields to send back to the generator throughout the run. -``gen_specs["in"]`` only describes the input fields when the function is **first called**. +Use ``GenSpecs.persis_in`` to specify fields to send back to the generator throughout the run. +``GenSpecs.inputs`` only describes the input fields when the function is **first called**. Functions for a persistent generator to communicate directly with the manager are available in the :ref:`libensemble.tools.persistent_support` class. @@ -168,7 +189,7 @@ a worker can be initiated in *active receive* mode by the allocation function (see :ref:`start_only_persistent`). The persistent worker can then send and receive from the manager at any time. -Ensure there are no communication deadlocks in this mode. In manager--worker message exchanges, only the worker-side +Ensure there are no communication deadlocks in this mode. In manager-worker message exchanges, only the worker-side receive is blocking by default (a non-blocking option is available). Cancelling Simulations diff --git a/docs/function_guides/simulator.rst b/docs/function_guides/simulator.rst index 35da79226..745cb9ae5 100644 --- a/docs/function_guides/simulator.rst +++ b/docs/function_guides/simulator.rst @@ -5,16 +5,41 @@ Simulator Functions Simulator and :ref:`Generator functions` have relatively similar interfaces. -.. code-block:: python +Writing a Simulator +------------------- - def my_simulation(Input, persis_info, sim_specs, libE_info): - batch_size = sim_specs["user"]["batch_size"] +.. tab-set:: + + .. tab-item:: Non-decorated + :sync: nodecorate + + .. code-block:: python + + def my_simulation(Input, persis_info, sim_specs, libE_info): + batch_size = sim_specs["user"]["batch_size"] + + Output = np.zeros(batch_size, sim_specs["out"]) + # ... + Output["f"], persis_info = do_a_simulation(Input["x"], persis_info) + + return Output, persis_info + + .. tab-item:: Decorated + :sync: decorate + + .. code-block:: python + + @input_fields(["x"]) + @output_data([("f", float)]) + def my_simulation(Input, persis_info, sim_specs, libE_info): + batch_size = sim_specs["user"]["batch_size"] - Output = np.zeros(batch_size, sim_specs["out"]) - ... - Output["f"], persis_info = do_a_simulation(Input["x"], persis_info) + Output = np.zeros(batch_size, sim_specs["out"]) + # ... + Output["f"], persis_info = do_a_simulation(Input["x"], persis_info) + + return Output, persis_info - return Output, persis_info Most ``sim_f`` function definitions written by users resemble:: @@ -22,25 +47,33 @@ Most ``sim_f`` function definitions written by users resemble:: where: - * ``Input`` is a selection of the :ref:`History array` - * :ref:`persis_info` is a dictionary containing state information - * :ref:`sim_specs` is a dictionary of simulation parameters, including which fields from the History array got sent - * ``libE_info`` is a dictionary containing libEnsemble-specific entries + * ``Input`` is a selection of the :ref:`History array`, a NumPy array. + * :ref:`persis_info` is a dictionary containing state information. + * :ref:`sim_specs` is a dictionary of simulation parameters. + * ``libE_info`` is a dictionary containing libEnsemble-specific entries. Valid simulator functions can accept a subset of the above parameters. So a very simple simulator function can start:: def my_simulation(Input): -If sim_specs was initially defined:: +If ``sim_specs`` was initially defined: + +.. tab-set:: + + .. tab-item:: Non-decorated function + :sync: nodecorate + + .. code-block:: python + + sim_specs = SimSpecs(sim_f=my_simulation, inputs=["x"], outputs=["f", float, (1,)], user={"batch_size": 128}) + + .. tab-item:: Decorated function + :sync: decorate + + .. code-block:: python + + sim_specs = SimSpecs(sim_f=my_simulation, user={"batch_size": 128}) - sim_specs = { - "sim_f": some_function, - "in": ["x"], - "out:" ["f", float, (1,)], - "user": { - "batch_size": 128 - } - } Then user parameters and a *local* array of outputs may be obtained/initialized like:: @@ -55,10 +88,9 @@ Then return the array and ``persis_info`` to libEnsemble:: return Output, persis_info -Between the ``Output`` definition and the ``return``, any level and complexity -of computation can be performed. Users are encouraged to use the :doc:`executor<../executor/overview>` -to submit applications to parallel resources if necessary, or plug in components from -other libraries to serve their needs. +Between the ``Output`` definition and the ``return``, any computation can be performed. +Users can try an :doc:`executor<../executor/overview>` to submit applications to parallel +resources, or plug in components from other libraries to serve their needs. Executor -------- @@ -73,7 +105,7 @@ for an additional example to try out. Persistent Simulators --------------------- -Although comparatively uncommon, simulator functions can also be written +Simulator functions can also be written in a persistent fashion. See the :ref:`here` for a general API overview of writing persistent generators, since the interface is largely identical. The only differences are to pass ``EVAL_SIM_TAG`` when instantiating a ``PersistentSupport`` From f47adb80d48a8065ac548cc7de877db6b30cc372 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 9 Oct 2023 13:16:58 -0500 Subject: [PATCH 08/10] add import line for decorators, fix specs formatting --- docs/function_guides/generator.rst | 15 +++++++++++++-- docs/function_guides/simulator.rst | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/function_guides/generator.rst b/docs/function_guides/generator.rst index 7f7a7bcac..6765dbd95 100644 --- a/docs/function_guides/generator.rst +++ b/docs/function_guides/generator.rst @@ -29,6 +29,9 @@ Writing a Generator .. code-block:: python + from libensemble.specs import input_fields, output_data + + @input_fields(["f"]) @output_data([("x", float)]) def my_generator(Input, persis_info, gen_specs, libE_info): @@ -64,14 +67,22 @@ If ``gen_specs`` was initially defined: .. code-block:: python - gen_specs = GenSpecs(gen_f=my_generator, inputs=["f"], outputs=["x", float, (1,)], user={"batch_size": 128}) + gen_specs = GenSpecs( + gen_f=my_generator, + inputs=["f"], + outputs=["x", float, (1,)], + user={"batch_size": 128}, + ) .. tab-item:: Decorated function :sync: decorate .. code-block:: python - gen_specs = GenSpecs(gen_f=my_generator, user={"batch_size": 128}) + gen_specs = GenSpecs( + gen_f=my_generator, + user={"batch_size": 128}, + ) Then user parameters and a *local* array of outputs may be obtained/initialized like:: diff --git a/docs/function_guides/simulator.rst b/docs/function_guides/simulator.rst index 745cb9ae5..9b6f92287 100644 --- a/docs/function_guides/simulator.rst +++ b/docs/function_guides/simulator.rst @@ -29,6 +29,9 @@ Writing a Simulator .. code-block:: python + from libensemble.specs import input_fields, output_data + + @input_fields(["x"]) @output_data([("f", float)]) def my_simulation(Input, persis_info, sim_specs, libE_info): @@ -65,14 +68,22 @@ If ``sim_specs`` was initially defined: .. code-block:: python - sim_specs = SimSpecs(sim_f=my_simulation, inputs=["x"], outputs=["f", float, (1,)], user={"batch_size": 128}) + sim_specs = SimSpecs( + sim_f=my_simulation, + inputs=["x"], + outputs=["f", float, (1,)], + user={"batch_size": 128}, + ) .. tab-item:: Decorated function :sync: decorate .. code-block:: python - sim_specs = SimSpecs(sim_f=my_simulation, user={"batch_size": 128}) + sim_specs = SimSpecs( + sim_f=my_simulation, + user={"batch_size": 128}, + ) Then user parameters and a *local* array of outputs may be obtained/initialized like:: From b539d71b000c7f46d7b62586a4cee89b7e155b8b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Dec 2023 11:33:44 -0600 Subject: [PATCH 09/10] re-add / implement attribute-setting validators for pydantic 1/2 models --- libensemble/utils/pydantic_bindings.py | 14 ++++++-- libensemble/utils/validators.py | 48 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/libensemble/utils/pydantic_bindings.py b/libensemble/utils/pydantic_bindings.py index 8277e3ac4..511d3d3f9 100644 --- a/libensemble/utils/pydantic_bindings.py +++ b/libensemble/utils/pydantic_bindings.py @@ -20,8 +20,10 @@ check_valid_in, check_valid_out, enable_save_H_when_every_K, + genf_set_in_out_from_attrs, set_platform_specs_to_class, set_workflow_dir, + simf_set_in_out_from_attrs, ) if pydanticV1: @@ -91,13 +93,21 @@ class Config: specs.SimSpecs = create_model( "SimSpecs", __base__=specs.SimSpecs, - __validators__={"check_valid_out": check_valid_out, "check_valid_in": check_valid_in}, + __validators__={ + "check_valid_out": check_valid_out, + "check_valid_in": check_valid_in, + "simf_set_in_out_from_attrs": simf_set_in_out_from_attrs, + }, ) specs.GenSpecs = create_model( "GenSpecs", __base__=specs.GenSpecs, - __validators__={"check_valid_out": check_valid_out, "check_valid_in": check_valid_in}, + __validators__={ + "check_valid_out": check_valid_out, + "check_valid_in": check_valid_in, + "genf_set_in_out_from_attrs": genf_set_in_out_from_attrs, + }, ) specs.LibeSpecs = create_model( diff --git a/libensemble/utils/validators.py b/libensemble/utils/validators.py index 12ebd1854..763ef78c0 100644 --- a/libensemble/utils/validators.py +++ b/libensemble/utils/validators.py @@ -136,6 +136,34 @@ def check_provided_ufuncs(cls, values): return values + @root_validator + def simf_set_in_out_from_attrs(cls, values): + if not values.get("sim_f"): + from libensemble.sim_funcs.one_d_func import one_d_example + + values["sim_f"] = one_d_example + if hasattr(values.get("sim_f"), "inputs") and not values.get("inputs"): + values["inputs"] = values.get("sim_f").inputs + if hasattr(values.get("sim_f"), "outputs") and not values.get("outputs"): + values["out"] = values.get("sim_f").outputs + if hasattr(values.get("sim_f"), "persis_in") and not values.get("persis_in"): + values["persis_in"] = values.get("sim_f").persis_in + return values + + @root_validator + def genf_set_in_out_from_attrs(cls, values): + if not values.get("gen_f"): + from libensemble.gen_funcs.sampling import latin_hypercube_sample + + values["gen_f"] = latin_hypercube_sample + if hasattr(values.get("gen_f"), "inputs") and not values.get("inputs"): + values["inputs"] = values.get("gen_f").inputs + if hasattr(values.get("gen_f"), "outputs") and not values.get("outputs"): + values["out"] = values.get("gen_f").outputs + if hasattr(values.get("gen_f"), "persis_in") and not values.get("persis_in"): + values["persis_in"] = values.get("gen_f").persis_in + return values + # RESOURCES VALIDATORS ##### @root_validator @@ -197,6 +225,26 @@ def check_provided_ufuncs(self): return self + @model_validator(mode="after") + def simf_set_in_out_from_attrs(self): + if hasattr(self.__dict__.get("sim_f"), "inputs") and not self.__dict__.get("inputs"): + self.__dict__["inputs"] = self.__dict__.get("sim_f").inputs + if hasattr(self.__dict__.get("sim_f"), "outputs") and not self.__dict__.get("outputs"): + self.__dict__["out"] = self.__dict__.get("sim_f").outputs + if hasattr(self.__dict__.get("sim_f"), "persis_in") and not self.__dict__.get("persis_in"): + self.__dict__["persis_in"] = self.__dict__.get("sim_f").persis_in + return self + + @model_validator(mode="after") + def genf_set_in_out_from_attrs(self): + if hasattr(self.__dict__.get("gen_f"), "inputs") and not self.__dict__.get("inputs"): + self.__dict__["inputs"] = self.__dict__.get("gen_f").inputs + if hasattr(self.__dict__.get("gen_f"), "outputs") and not self.__dict__.get("outputs"): + self.__dict__["out"] = self.__dict__.get("gen_f").outputs + if hasattr(self.__dict__.get("gen_f"), "persis_in") and not self.__dict__.get("persis_in"): + self.__dict__["persis_in"] = self.__dict__.get("gen_f").persis_in + return self + # RESOURCES VALIDATORS ##### @model_validator(mode="after") From cdfd5d37f74647abea4c8425adb6e6cf093ff9a5 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Dec 2023 11:52:07 -0600 Subject: [PATCH 10/10] bugfix --- libensemble/utils/validators.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/utils/validators.py b/libensemble/utils/validators.py index 763ef78c0..1057e0020 100644 --- a/libensemble/utils/validators.py +++ b/libensemble/utils/validators.py @@ -145,7 +145,7 @@ def simf_set_in_out_from_attrs(cls, values): if hasattr(values.get("sim_f"), "inputs") and not values.get("inputs"): values["inputs"] = values.get("sim_f").inputs if hasattr(values.get("sim_f"), "outputs") and not values.get("outputs"): - values["out"] = values.get("sim_f").outputs + values["outputs"] = values.get("sim_f").outputs if hasattr(values.get("sim_f"), "persis_in") and not values.get("persis_in"): values["persis_in"] = values.get("sim_f").persis_in return values @@ -159,7 +159,7 @@ def genf_set_in_out_from_attrs(cls, values): if hasattr(values.get("gen_f"), "inputs") and not values.get("inputs"): values["inputs"] = values.get("gen_f").inputs if hasattr(values.get("gen_f"), "outputs") and not values.get("outputs"): - values["out"] = values.get("gen_f").outputs + values["outputs"] = values.get("gen_f").outputs if hasattr(values.get("gen_f"), "persis_in") and not values.get("persis_in"): values["persis_in"] = values.get("gen_f").persis_in return values @@ -230,7 +230,7 @@ def simf_set_in_out_from_attrs(self): if hasattr(self.__dict__.get("sim_f"), "inputs") and not self.__dict__.get("inputs"): self.__dict__["inputs"] = self.__dict__.get("sim_f").inputs if hasattr(self.__dict__.get("sim_f"), "outputs") and not self.__dict__.get("outputs"): - self.__dict__["out"] = self.__dict__.get("sim_f").outputs + self.__dict__["outputs"] = self.__dict__.get("sim_f").outputs if hasattr(self.__dict__.get("sim_f"), "persis_in") and not self.__dict__.get("persis_in"): self.__dict__["persis_in"] = self.__dict__.get("sim_f").persis_in return self @@ -240,7 +240,7 @@ def genf_set_in_out_from_attrs(self): if hasattr(self.__dict__.get("gen_f"), "inputs") and not self.__dict__.get("inputs"): self.__dict__["inputs"] = self.__dict__.get("gen_f").inputs if hasattr(self.__dict__.get("gen_f"), "outputs") and not self.__dict__.get("outputs"): - self.__dict__["out"] = self.__dict__.get("gen_f").outputs + self.__dict__["outputs"] = self.__dict__.get("gen_f").outputs if hasattr(self.__dict__.get("gen_f"), "persis_in") and not self.__dict__.get("persis_in"): self.__dict__["persis_in"] = self.__dict__.get("gen_f").persis_in return self