diff --git a/docs/function_guides/generator.rst b/docs/function_guides/generator.rst index c8d0b3ef6..6765dbd95 100644 --- a/docs/function_guides/generator.rst +++ b/docs/function_guides/generator.rst @@ -5,16 +5,43 @@ Generator Functions Generator and :ref:`Simulator functions` have relatively similar interfaces. -.. code-block:: python +Writing a Generator +------------------- - def my_generator(Input, persis_info, gen_specs, libE_info): - batch_size = gen_specs["user"]["batch_size"] +.. tab-set:: + + .. tab-item:: Non-decorated + :sync: nodecorate + + .. code-block:: python + + def my_generator(Input, persis_info, gen_specs, libE_info): + batch_size = gen_specs["user"]["batch_size"] + + Output = np.zeros(batch_size, gen_specs["out"]) + # ... + Output["x"], persis_info = generate_next_simulation_inputs(Input["f"], persis_info) + + return Output, persis_info + + .. tab-item:: Decorated + :sync: decorate + + .. code-block:: python - Output = np.zeros(batch_size, gen_specs["out"]) - ... - Output["x"], persis_info = generate_next_simulation_inputs(Input["f"], persis_info) + from libensemble.specs import input_fields, output_data - return Output, persis_info + + @input_fields(["f"]) + @output_data([("x", float)]) + def my_generator(Input, persis_info, gen_specs, libE_info): + batch_size = gen_specs["user"]["batch_size"] + + Output = np.zeros(batch_size, gen_specs["out"]) + # ... + Output["x"], persis_info = generate_next_simulation_inputs(Input["f"], persis_info) + + return Output, persis_info Most ``gen_f`` function definitions written by users resemble:: @@ -22,25 +49,40 @@ Most ``gen_f`` function definitions written by users resemble:: where: - * ``Input`` is a selection of the :ref:`History array` - * :ref:`persis_info` is a dictionary containing state information - * :ref:`gen_specs` is a dictionary of generator parameters, including which fields from the History array got sent - * ``libE_info`` is a dictionary containing libEnsemble-specific entries + * ``Input`` is a selection of the :ref:`History array`, a NumPy array. + * :ref:`persis_info` is a dictionary containing state information. + * :ref:`gen_specs` is a dictionary of generator parameters. + * ``libE_info`` is a dictionary containing miscellaneous entries. Valid generator functions can accept a subset of the above parameters. So a very simple generator can start:: def my_generator(Input): -If gen_specs was initially defined:: +If ``gen_specs`` was initially defined: + +.. tab-set:: + + .. tab-item:: Non-decorated function + :sync: nodecorate + + .. code-block:: python + + gen_specs = GenSpecs( + gen_f=my_generator, + inputs=["f"], + outputs=["x", float, (1,)], + user={"batch_size": 128}, + ) + + .. tab-item:: Decorated function + :sync: decorate + + .. code-block:: python - gen_specs = { - "gen_f": some_function, - "in": ["f"], - "out:" ["x", float, (1,)], - "user": { - "batch_size": 128 - } - } + gen_specs = GenSpecs( + gen_f=my_generator, + user={"batch_size": 128}, + ) Then user parameters and a *local* array of outputs may be obtained/initialized like:: @@ -56,10 +98,9 @@ Then return the array and ``persis_info`` to libEnsemble:: return Output, persis_info -Between the ``Output`` definition and the ``return``, any level and complexity -of computation can be performed. Users are encouraged to use the :doc:`executor<../executor/overview>` -to submit applications to parallel resources if necessary, or plug in components from -other libraries to serve their needs. +Between the ``Output`` definition and the ``return``, any computation can be performed. +Users can try an :doc:`executor<../executor/overview>` to submit applications to parallel +resources, or plug in components from other libraries to serve their needs. .. note:: @@ -74,17 +115,17 @@ Persistent Generators While non-persistent generators return after completing their calculation, persistent generators do the following in a loop: - 1. Receive simulation results and metadata; exit if metadata instructs - 2. Perform analysis - 3. Send subsequent simulation parameters + 1. Receive simulation results and metadata; exit if metadata instructs. + 2. Perform analysis. + 3. Send subsequent simulation parameters. Persistent generators don't need to be re-initialized on each call, but are typically -more complicated. The :doc:`APOSMM<../examples/aposmm>` -optimization generator function included with libEnsemble is persistent so it can -maintain multiple local optimization subprocesses based on results from complete simulations. +more complicated. The persistent :doc:`APOSMM<../examples/aposmm>` +optimization generator function included with libEnsemble maintains +local optimization subprocesses based on results from complete simulations. -Use ``gen_specs["persis_in"]`` to specify fields to send back to the generator throughout the run. -``gen_specs["in"]`` only describes the input fields when the function is **first called**. +Use ``GenSpecs.persis_in`` to specify fields to send back to the generator throughout the run. +``GenSpecs.inputs`` only describes the input fields when the function is **first called**. Functions for a persistent generator to communicate directly with the manager are available in the :ref:`libensemble.tools.persistent_support` class. @@ -159,7 +200,7 @@ a worker can be initiated in *active receive* mode by the allocation function (see :ref:`start_only_persistent`). The persistent worker can then send and receive from the manager at any time. -Ensure there are no communication deadlocks in this mode. In manager--worker message exchanges, only the worker-side +Ensure there are no communication deadlocks in this mode. In manager-worker message exchanges, only the worker-side receive is blocking by default (a non-blocking option is available). Cancelling Simulations diff --git a/docs/function_guides/simulator.rst b/docs/function_guides/simulator.rst index de8092abf..5d28de696 100644 --- a/docs/function_guides/simulator.rst +++ b/docs/function_guides/simulator.rst @@ -5,16 +5,44 @@ Simulator Functions Simulator and :ref:`Generator functions` have relatively similar interfaces. -.. code-block:: python +Writing a Simulator +------------------- + +.. tab-set:: + + .. tab-item:: Non-decorated + :sync: nodecorate + + .. code-block:: python + + def my_simulation(Input, persis_info, sim_specs, libE_info): + batch_size = sim_specs["user"]["batch_size"] + + Output = np.zeros(batch_size, sim_specs["out"]) + # ... + Output["f"], persis_info = do_a_simulation(Input["x"], persis_info) + + return Output, persis_info + + .. tab-item:: Decorated + :sync: decorate + + .. code-block:: python + + from libensemble.specs import input_fields, output_data - def my_simulation(Input, persis_info, sim_specs, libE_info): - batch_size = sim_specs["user"]["batch_size"] - Output = np.zeros(batch_size, sim_specs["out"]) - ... - Output["f"], persis_info = do_a_simulation(Input["x"], persis_info) + @input_fields(["x"]) + @output_data([("f", float)]) + def my_simulation(Input, persis_info, sim_specs, libE_info): + batch_size = sim_specs["user"]["batch_size"] + + Output = np.zeros(batch_size, sim_specs["out"]) + # ... + Output["f"], persis_info = do_a_simulation(Input["x"], persis_info) + + return Output, persis_info - return Output, persis_info Most ``sim_f`` function definitions written by users resemble:: @@ -22,25 +50,41 @@ Most ``sim_f`` function definitions written by users resemble:: where: - * ``Input`` is a selection of the :ref:`History array` - * :ref:`persis_info` is a dictionary containing state information - * :ref:`sim_specs` is a dictionary of simulation parameters, including which fields from the History array got sent - * ``libE_info`` is a dictionary containing libEnsemble-specific entries + * ``Input`` is a selection of the :ref:`History array`, a NumPy array. + * :ref:`persis_info` is a dictionary containing state information. + * :ref:`sim_specs` is a dictionary of simulation parameters. + * ``libE_info`` is a dictionary containing libEnsemble-specific entries. Valid simulator functions can accept a subset of the above parameters. So a very simple simulator function can start:: def my_simulation(Input): -If sim_specs was initially defined:: +If ``sim_specs`` was initially defined: + +.. tab-set:: + + .. tab-item:: Non-decorated function + :sync: nodecorate + + .. code-block:: python + + sim_specs = SimSpecs( + sim_f=my_simulation, + inputs=["x"], + outputs=["f", float, (1,)], + user={"batch_size": 128}, + ) + + .. tab-item:: Decorated function + :sync: decorate + + .. code-block:: python + + sim_specs = SimSpecs( + sim_f=my_simulation, + user={"batch_size": 128}, + ) - sim_specs = { - "sim_f": some_function, - "in": ["x"], - "out:" ["f", float, (1,)], - "user": { - "batch_size": 128 - } - } Then user parameters and a *local* array of outputs may be obtained/initialized like:: @@ -55,10 +99,9 @@ Then return the array and ``persis_info`` to libEnsemble:: return Output, persis_info -Between the ``Output`` definition and the ``return``, any level and complexity -of computation can be performed. Users are encouraged to use the :doc:`executor<../executor/overview>` -to submit applications to parallel resources if necessary, or plug in components from -other libraries to serve their needs. +Between the ``Output`` definition and the ``return``, any computation can be performed. +Users can try an :doc:`executor<../executor/overview>` to submit applications to parallel +resources, or plug in components from other libraries to serve their needs. Executor -------- @@ -73,7 +116,7 @@ for an additional example to try out. Persistent Simulators --------------------- -Although comparatively uncommon, simulator functions can also be written +Simulator functions can also be written in a persistent fashion. See the :ref:`here` for a general API overview of writing persistent generators, since the interface is largely identical. The only differences are to pass ``EVAL_SIM_TAG`` when instantiating a ``PersistentSupport`` diff --git a/libensemble/sim_funcs/one_d_func.py b/libensemble/sim_funcs/one_d_func.py index 61d2f45f7..484288b92 100644 --- a/libensemble/sim_funcs/one_d_func.py +++ b/libensemble/sim_funcs/one_d_func.py @@ -5,7 +5,11 @@ import numpy as np +from libensemble.specs import input_fields, output_data + +@input_fields(["x"]) +@output_data([("f", float)]) def one_d_example(x, persis_info, sim_specs, _): """ Evaluates the six hump camel function for a single point ``x``. diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index c1c9f7176..85550fd50 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -17,9 +17,12 @@ import numpy as np from libensemble.message_numbers import EVAL_SIM_TAG, FINISHED_PERSISTENT_SIM_TAG, PERSIS_STOP, STOP_TAG +from libensemble.specs import input_fields, output_data from libensemble.tools.persistent_support import PersistentSupport +@input_fields(["x"]) +@output_data([("f", float)]) def six_hump_camel(H, persis_info, sim_specs, libE_info): """ Evaluates the six hump camel function for a collection of points given in ``H["x"]``. diff --git a/libensemble/specs.py b/libensemble/specs.py index 5b88e20da..bd431f316 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -39,7 +39,7 @@ class SimSpecs(BaseModel): """ # list of tuples for dtype construction - outputs: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]] = Field(default=[], alias="out") + outputs: Optional[List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]] = Field([], alias="out") """ List of 2- or 3-tuples corresponding to NumPy dtypes. e.g. ``("dim", int, (3,))``, or ``("path", str)``. @@ -85,7 +85,7 @@ class GenSpecs(BaseModel): throughout the run, following initialization. """ - outputs: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]] = Field(default=[], alias="out") + outputs: Optional[List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]] = Field([], alias="out") """ List of 2- or 3-tuples corresponding to NumPy dtypes. e.g. ``("dim", int, (3,))``, or ``("path", str)``. Typically used to initialize an @@ -470,3 +470,91 @@ class _EnsembleSpecs(BaseModel): alloc_specs: Optional[AllocSpecs] = AllocSpecs() """ Specifications for the allocation function. """ + + +def input_fields(fields: List[str]): + """Decorates a user-function with a list of field names to pass in on initialization. + + Decorated functions don't need those fields specified in ``SimSpecs.inputs`` or ``GenSpecs.inputs``. + + .. code-block:: python + + from libensemble.specs import input_fields, output_data + + + @input_fields(["x"]) + @output_data([("f", float)]) + def one_d_example(x, persis_info, sim_specs): + H_o = np.zeros(1, dtype=sim_specs["out"]) + H_o["f"] = np.linalg.norm(x) + return H_o, persis_info + """ + + def decorator(func): + setattr(func, "inputs", fields) + func.__doc__ = f"\n **Input Fields:** ``{func.inputs}``\n" + func.__doc__ + return func + + return decorator + + +def persistent_input_fields(fields: List[str]): + """Decorates a *persistent* user-function with a list of field names to send in throughout runtime. + + Decorated functions don't need those fields specified in ``SimSpecs.persis_in`` or ``GenSpecs.persis_in``. + + .. code-block:: python + + from libensemble.specs import persistent_input_fields, output_data + + + @persistent_input_fields(["f"]) + @output_data(["x", float]) + def persistent_uniform(_, persis_info, gen_specs, libE_info): + + b, n, lb, ub = _get_user_params(gen_specs["user"]) + ps = PersistentSupport(libE_info, EVAL_GEN_TAG) + + tag = None + while tag not in [STOP_TAG, PERSIS_STOP]: + H_o = np.zeros(b, dtype=gen_specs["out"]) + H_o["x"] = persis_info["rand_stream"].uniform(lb, ub, (b, n)) + tag, Work, calc_in = ps.send_recv(H_o) + if hasattr(calc_in, "__len__"): + b = len(calc_in) + + return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + """ + + def decorator(func): + setattr(func, "persis_in", fields) + func.__doc__ = f"\n **Persistent Input Fields:** ``{func.persis_inputs}``\n" + func.__doc__ + return func + + return decorator + + +def output_data(fields: List[Union[Tuple[str, Any], Tuple[str, Any, Union[int, Tuple]]]]): + """Decorates a user-function with a list of tuples corresponding to NumPy dtypes for the function's output data. + + Decorated functions don't need those fields specified in ``SimSpecs.outputs`` or ``GenSpecs.outputs``. + + .. code-block:: python + + from libensemble.specs import input_fields, output_data + + + @input_fields(["x"]) + @output_data([("f", float)]) + def one_d_example(x, persis_info, sim_specs): + H_o = np.zeros(1, dtype=sim_specs["out"]) + H_o["f"] = np.linalg.norm(x) + return H_o, persis_info + """ + + def decorator(func): + setattr(func, "outputs", fields) + func.__doc__ = f"\n **Output Datatypes:** ``{func.outputs}``\n" + func.__doc__ + return func + + return decorator diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index 4747f9d80..e0ac99dca 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -26,18 +26,8 @@ # Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). if __name__ == "__main__": sampling = Ensemble(parse_args=True) - sampling.libE_specs = LibeSpecs( - save_every_k_gens=300, - safe_mode=False, - disable_log_files=True, - ) - - sampling.sim_specs = SimSpecs( - sim_f=sim_f, - inputs=["x"], - outputs=[("f", float)], - ) - + sampling.libE_specs = LibeSpecs(save_every_k_gens=300, safe_mode=False, disable_log_files=True) + sampling.sim_specs = SimSpecs(sim_f=sim_f) sampling.gen_specs = GenSpecs( gen_f=gen_f, outputs=[("x", float, (1,))], diff --git a/libensemble/utils/pydantic_bindings.py b/libensemble/utils/pydantic_bindings.py index 8277e3ac4..511d3d3f9 100644 --- a/libensemble/utils/pydantic_bindings.py +++ b/libensemble/utils/pydantic_bindings.py @@ -20,8 +20,10 @@ check_valid_in, check_valid_out, enable_save_H_when_every_K, + genf_set_in_out_from_attrs, set_platform_specs_to_class, set_workflow_dir, + simf_set_in_out_from_attrs, ) if pydanticV1: @@ -91,13 +93,21 @@ class Config: specs.SimSpecs = create_model( "SimSpecs", __base__=specs.SimSpecs, - __validators__={"check_valid_out": check_valid_out, "check_valid_in": check_valid_in}, + __validators__={ + "check_valid_out": check_valid_out, + "check_valid_in": check_valid_in, + "simf_set_in_out_from_attrs": simf_set_in_out_from_attrs, + }, ) specs.GenSpecs = create_model( "GenSpecs", __base__=specs.GenSpecs, - __validators__={"check_valid_out": check_valid_out, "check_valid_in": check_valid_in}, + __validators__={ + "check_valid_out": check_valid_out, + "check_valid_in": check_valid_in, + "genf_set_in_out_from_attrs": genf_set_in_out_from_attrs, + }, ) specs.LibeSpecs = create_model( diff --git a/libensemble/utils/validators.py b/libensemble/utils/validators.py index 12ebd1854..1057e0020 100644 --- a/libensemble/utils/validators.py +++ b/libensemble/utils/validators.py @@ -136,6 +136,34 @@ def check_provided_ufuncs(cls, values): return values + @root_validator + def simf_set_in_out_from_attrs(cls, values): + if not values.get("sim_f"): + from libensemble.sim_funcs.one_d_func import one_d_example + + values["sim_f"] = one_d_example + if hasattr(values.get("sim_f"), "inputs") and not values.get("inputs"): + values["inputs"] = values.get("sim_f").inputs + if hasattr(values.get("sim_f"), "outputs") and not values.get("outputs"): + values["outputs"] = values.get("sim_f").outputs + if hasattr(values.get("sim_f"), "persis_in") and not values.get("persis_in"): + values["persis_in"] = values.get("sim_f").persis_in + return values + + @root_validator + def genf_set_in_out_from_attrs(cls, values): + if not values.get("gen_f"): + from libensemble.gen_funcs.sampling import latin_hypercube_sample + + values["gen_f"] = latin_hypercube_sample + if hasattr(values.get("gen_f"), "inputs") and not values.get("inputs"): + values["inputs"] = values.get("gen_f").inputs + if hasattr(values.get("gen_f"), "outputs") and not values.get("outputs"): + values["outputs"] = values.get("gen_f").outputs + if hasattr(values.get("gen_f"), "persis_in") and not values.get("persis_in"): + values["persis_in"] = values.get("gen_f").persis_in + return values + # RESOURCES VALIDATORS ##### @root_validator @@ -197,6 +225,26 @@ def check_provided_ufuncs(self): return self + @model_validator(mode="after") + def simf_set_in_out_from_attrs(self): + if hasattr(self.__dict__.get("sim_f"), "inputs") and not self.__dict__.get("inputs"): + self.__dict__["inputs"] = self.__dict__.get("sim_f").inputs + if hasattr(self.__dict__.get("sim_f"), "outputs") and not self.__dict__.get("outputs"): + self.__dict__["outputs"] = self.__dict__.get("sim_f").outputs + if hasattr(self.__dict__.get("sim_f"), "persis_in") and not self.__dict__.get("persis_in"): + self.__dict__["persis_in"] = self.__dict__.get("sim_f").persis_in + return self + + @model_validator(mode="after") + def genf_set_in_out_from_attrs(self): + if hasattr(self.__dict__.get("gen_f"), "inputs") and not self.__dict__.get("inputs"): + self.__dict__["inputs"] = self.__dict__.get("gen_f").inputs + if hasattr(self.__dict__.get("gen_f"), "outputs") and not self.__dict__.get("outputs"): + self.__dict__["outputs"] = self.__dict__.get("gen_f").outputs + if hasattr(self.__dict__.get("gen_f"), "persis_in") and not self.__dict__.get("persis_in"): + self.__dict__["persis_in"] = self.__dict__.get("gen_f").persis_in + return self + # RESOURCES VALIDATORS ##### @model_validator(mode="after")