From a5d949e7cc6f3e9c1555d4f2eb0848995d26e31a Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Sat, 10 Aug 2024 13:06:10 -0600 Subject: [PATCH 01/11] First attempt at new params-to-vars code. --- pyomo/contrib/parmest/parmest.py | 7 +- pyomo/contrib/parmest/tests/test_utils.py | 47 +++++- pyomo/contrib/parmest/utils/model_utils.py | 185 ++++++++++++++++++++- 3 files changed, 232 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/parmest/parmest.py b/pyomo/contrib/parmest/parmest.py index 41e7792570b..1dc0b8dc66d 100644 --- a/pyomo/contrib/parmest/parmest.py +++ b/pyomo/contrib/parmest/parmest.py @@ -440,8 +440,8 @@ def TotalCost_rule(model): ) # Convert theta Params to Vars, and unfix theta Vars - theta_names = [k.name for k, v in model.unknown_parameters.items()] - parmest_model = utils.convert_params_to_vars(model, theta_names, fix_vars=False) + theta_CUIDs = [v for k, v in model.unknown_parameters.items()] + parmest_model = utils.convert_params_to_vars(model, theta_CUIDs, fix_vars=False) return parmest_model @@ -1556,7 +1556,8 @@ def TotalCost_rule(model): ) # Convert theta Params to Vars, and unfix theta Vars - model = utils.convert_params_to_vars(model, self.theta_names) + theta_CUIDs = [ComponentUID(theta_name) for theta_name in self.theta_names] + model = utils.convert_params_to_vars(model, theta_CUIDs) # Update theta names list to use CUID string representation for i, theta in enumerate(self.theta_names): diff --git a/pyomo/contrib/parmest/tests/test_utils.py b/pyomo/contrib/parmest/tests/test_utils.py index d5e66ab58d5..6cbaddb1c62 100644 --- a/pyomo/contrib/parmest/tests/test_utils.py +++ b/pyomo/contrib/parmest/tests/test_utils.py @@ -29,6 +29,9 @@ class TestUtils(unittest.TestCase): def test_convert_param_to_var(self): # TODO: Check that this works for different structured models (indexed, blocks, etc) + # test params + ############# + from pyomo.contrib.parmest.examples.reactor_design.reactor_design import ( ReactorDesignExperiment, ) @@ -46,12 +49,12 @@ def test_convert_param_to_var(self): exp = ReactorDesignExperiment(data, 0) instance = exp.get_labeled_model() - theta_names = ['k1', 'k2', 'k3'] + param_CUIDs = [v for k, v in instance.unknown_parameters.items()] m_vars = parmest.utils.convert_params_to_vars( - instance, theta_names, fix_vars=True + instance, param_CUIDs, fix_vars=True ) - for v in theta_names: + for v in [str(CUID) for CUID in param_CUIDs]: self.assertTrue(hasattr(m_vars, v)) c = m_vars.find_component(v) self.assertIsInstance(c, pyo.Var) @@ -61,5 +64,43 @@ def test_convert_param_to_var(self): self.assertTrue(c in m_vars.unknown_parameters) + # test indexed params + ##################### + + from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, + ) + + self.data = pd.DataFrame( + data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0], [5, 15.6], [7, 19.8]], + columns=["hour", "y"], + ) + + class RooneyBieglerExperimentIndexedParams(RooneyBieglerExperiment): + + def create_model(self): + data_df = self.data.to_frame().transpose() + self.model = rooney_biegler_indexed_params(data_df) + + def label_model(self): + + m = self.model + + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update( + [(m.hour, self.data["hour"]), (m.y, self.data["y"])] + ) + + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update((k, pyo.ComponentUID(k)) for k in [m.theta]) + + rooney_biegler_indexed_params_exp_list = [] + for i in range(self.data.shape[0]): + rooney_biegler_indexed_params_exp_list.append( + RooneyBieglerExperimentIndexedParams(self.data.loc[i, :]) + ) + + + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 7778ebcc9f1..39c93d3ca5a 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -21,8 +21,191 @@ logger = logging.getLogger(__name__) +def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): + """ + Convert select Params to Vars + + Parameters + ---------- + model : Pyomo concrete model + Original model + param_CUIDs : list of CUIDs to convert, if None then all Params are converted + fix_vars : bool + Fix the new variables, default is False + + Returns + ------- + model : Pyomo concrete model + Model with select Params converted to Vars + """ + + model = model.clone() + + if param_CUIDs is None: + param_CUIDs = [ + ComponentUID(param) for param in model.component_data_objects(pyo.Param) + ] + + # keep a list of the parameter names for creating the new model + param_names = [str(param_CUID) for param_CUID in param_CUIDs] + + # Convert Params to Vars, unfix Vars, and create a substitution map + indexed_param_names = [] + substitution_map = {} + comp_map = ComponentMap() + for param_CUID in param_CUIDs: + + # Leverage the parser in ComponentUID to locate the component. + param_object = param_CUID.find_component_on(model) + + # Param + if param_object.is_parameter_type(): + + # Delete Param, add Var + vals = param_object.extract_values() + model.del_component(param_object) + model.add_component(param_object.name, pyo.Var(initialize=vals[None])) + + # Update substitution map + new_var_cuid = ComponentUID(param_object) + new_var_object = new_var_cuid.find_component_on(model) + substitution_map[id(param_object)] = new_var_object + comp_map[param_object] = new_var_object + + # Indexed Param + elif isinstance(param_object, IndexedParam): + + print('indexed param') + # Delete Param, add Var + # Before deleting the Param, create a list of the indexed param names + vals = param_object.extract_values() + param_objects = [] + for param_obj in param_object: + indexed_param_name = param_object.name + "[" + str(param_obj) + "]" + param_cuid = ComponentUID(indexed_param_name) + param_objects.append(param_cuid.find_component_on(model)) + indexed_param_names.append(indexed_param_name) + + model.del_component(param_object) + + index_name = param_object.index_set().name + index_cuid = ComponentUID(index_name) + index_object = index_cuid.find_component_on(model) + model.add_component( + param_object.name, pyo.Var(index_object, initialize=vals) + ) + + # Update substitution map (map each indexed param to indexed var) + new_var_cuid = ComponentUID(param_object.name) + new_var_object = new_var_cuid.find_component_on(model) + comp_map[param_object] = new_var_object + new_var_objects = [] + for var_obj in new_var_object: + var_cuid = ComponentUID(new_var_object.name + "[" + str(var_obj) + "]") + new_var_objects.append(var_cuid.find_component_on(model)) + + for param_obj, new_var_obj in zip(param_objects, new_var_objects): + substitution_map[id(param_obj)] = new_var_obj + comp_map[param_obj] = new_var_obj + + # Var or Indexed Var + elif isinstance(param_object, IndexedVar) or param_object.is_variable_type(): + new_var_object = param_object + + else: + logger.warning("%s is not a Param or Var on the model", (str(param_object))) + return model + + if fix_vars: + new_var_object.fix() + else: + new_var_object.unfix() + + # If no substitutions are needed, return the model + if len(substitution_map) == 0: + return model + + # Update the list of param_names if the parameters were indexed + if len(indexed_param_names) > 0: + param_names = indexed_param_names + + # Convert Params to Vars in Expressions + for expr in model.component_data_objects(pyo.Expression): + if expr.active and any( + ComponentUID(v) in param_CUIDs for v in identify_mutable_parameters(expr) + ): + new_expr = replace_expressions(expr=expr, substitution_map=substitution_map) + model.del_component(expr) + model.add_component(expr.name, pyo.Expression(rule=new_expr)) + + # Convert Params to Vars in Constraint expressions + num_constraints = len(list(model.component_objects(pyo.Constraint, active=True))) + if num_constraints > 0: + model.constraints = pyo.ConstraintList() + for c in model.component_data_objects(pyo.Constraint): + if c.active and any( + ComponentUID(v) in param_CUIDs for v in identify_mutable_parameters(c.expr) + ): + if c.equality: + model.constraints.add( + replace_expressions( + expr=c.lower, substitution_map=substitution_map + ) + == replace_expressions( + expr=c.body, substitution_map=substitution_map + ) + ) + elif c.lower is not None: + model.constraints.add( + replace_expressions( + expr=c.lower, substitution_map=substitution_map + ) + <= replace_expressions( + expr=c.body, substitution_map=substitution_map + ) + ) + elif c.upper is not None: + model.constraints.add( + replace_expressions( + expr=c.upper, substitution_map=substitution_map + ) + >= replace_expressions( + expr=c.body, substitution_map=substitution_map + ) + ) + else: + raise ValueError( + "Unable to parse constraint to convert params to vars." + ) + c.deactivate() + + # Convert Params to Vars in Objective expressions + for obj in model.component_data_objects(pyo.Objective): + if obj.active and any( + ComponentUID(v) in param_CUIDs for v in identify_mutable_parameters(obj) + ): + expr = replace_expressions(expr=obj.expr, substitution_map=substitution_map) + model.del_component(obj) + model.add_component(obj.name, pyo.Objective(rule=expr, sense=obj.sense)) + + # Convert Params to Vars in Suffixes + for s in model.component_objects(pyo.Suffix): + current_keys = list(s.keys()) + for c in current_keys: + if c in comp_map: + s[comp_map[c]] = s.pop(c) + + assert len(current_keys) == len(s.keys()) + + # print('--- Updated Model ---') + # model.pprint() + # solver = pyo.SolverFactory('ipopt') + # solver.solve(model) + + return model + -def convert_params_to_vars(model, param_names=None, fix_vars=False): +def convert_params_to_vars_deprecated(model, param_names=None, fix_vars=False): """ Convert select Params to Vars From e534e32421357f73c341d4aaf75ceb67a0c2a237 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Sat, 10 Aug 2024 15:24:19 -0600 Subject: [PATCH 02/11] Removed all strings from convert_params_to_vars in parmest utils. --- pyomo/contrib/parmest/tests/test_utils.py | 49 ++++++++-- pyomo/contrib/parmest/utils/model_utils.py | 102 +++++++++++---------- 2 files changed, 96 insertions(+), 55 deletions(-) diff --git a/pyomo/contrib/parmest/tests/test_utils.py b/pyomo/contrib/parmest/tests/test_utils.py index 6cbaddb1c62..bbfa90e9425 100644 --- a/pyomo/contrib/parmest/tests/test_utils.py +++ b/pyomo/contrib/parmest/tests/test_utils.py @@ -11,6 +11,7 @@ from pyomo.common.dependencies import pandas as pd, pandas_available +from pyomo.core.base.var import IndexedVar import pyomo.environ as pyo import pyomo.common.unittest as unittest import pyomo.contrib.parmest.parmest as parmest @@ -76,6 +77,29 @@ def test_convert_param_to_var(self): columns=["hour", "y"], ) + def rooney_biegler_indexed_params(data): + model = pyo.ConcreteModel() + + model.param_names = pyo.Set(initialize=["asymptote", "rate_constant"]) + model.theta = pyo.Param( + model.param_names, + initialize={"asymptote": 15, "rate_constant": 0.5}, + mutable=True, + ) + + model.hour = pyo.Param(within=pyo.PositiveReals, mutable=True) + model.y = pyo.Param(within=pyo.PositiveReals, mutable=True) + + def response_rule(m, h): + expr = m.theta["asymptote"] * ( + 1 - pyo.exp(-m.theta["rate_constant"] * h) + ) + return expr + + model.response_function = pyo.Expression(data.hour, rule=response_rule) + + return model + class RooneyBieglerExperimentIndexedParams(RooneyBieglerExperiment): def create_model(self): @@ -94,13 +118,24 @@ def label_model(self): m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) m.unknown_parameters.update((k, pyo.ComponentUID(k)) for k in [m.theta]) - rooney_biegler_indexed_params_exp_list = [] - for i in range(self.data.shape[0]): - rooney_biegler_indexed_params_exp_list.append( - RooneyBieglerExperimentIndexedParams(self.data.loc[i, :]) - ) - - + exp = RooneyBieglerExperimentIndexedParams(self.data.loc[0, :]) + instance = exp.get_labeled_model() + + param_CUIDs = [v for k, v in instance.unknown_parameters.items()] + m_vars = parmest.utils.convert_params_to_vars( + instance, param_CUIDs, fix_vars=True + ) + + for v in [str(CUID) for CUID in param_CUIDs]: + self.assertTrue(hasattr(m_vars, v)) + c = m_vars.find_component(v) + self.assertIsInstance(c, IndexedVar) + for _, iv in c.items(): + self.assertTrue(iv.fixed) + iv_old = instance.find_component(iv) + self.assertEqual(pyo.value(iv), pyo.value(iv_old)) + self.assertTrue(c in m_vars.unknown_parameters) + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 39c93d3ca5a..362a7c0a218 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) + def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): """ Convert select Params to Vars @@ -29,7 +30,8 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): ---------- model : Pyomo concrete model Original model - param_CUIDs : list of CUIDs to convert, if None then all Params are converted + param_CUIDs : list of strings + List of parameter CUIDs to convert, if None then all Params are converted fix_vars : bool Fix the new variables, default is False @@ -45,89 +47,93 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): param_CUIDs = [ ComponentUID(param) for param in model.component_data_objects(pyo.Param) ] - - # keep a list of the parameter names for creating the new model - param_names = [str(param_CUID) for param_CUID in param_CUIDs] + + # if param_CUIDs is None: + # param_CUIDs = [ + # ComponentUID(param.name) + # for param in model.component_data_objects(pyo.Param) + # ] + + # keep a list of the parameter CUIDs in the case of indexing + indexed_param_CUIDs = [] # Convert Params to Vars, unfix Vars, and create a substitution map - indexed_param_names = [] substitution_map = {} comp_map = ComponentMap() for param_CUID in param_CUIDs: # Leverage the parser in ComponentUID to locate the component. - param_object = param_CUID.find_component_on(model) + theta_object = param_CUID.find_component_on(model) # Param - if param_object.is_parameter_type(): + if theta_object.is_parameter_type(): # Delete Param, add Var - vals = param_object.extract_values() - model.del_component(param_object) - model.add_component(param_object.name, pyo.Var(initialize=vals[None])) + vals = theta_object.extract_values() + model.del_component(theta_object) + model.add_component(theta_object.name, pyo.Var(initialize=vals[None])) # Update substitution map - new_var_cuid = ComponentUID(param_object) - new_var_object = new_var_cuid.find_component_on(model) - substitution_map[id(param_object)] = new_var_object - comp_map[param_object] = new_var_object + theta_var_cuid = ComponentUID(theta_object.name) + theta_var_object = theta_var_cuid.find_component_on(model) + substitution_map[id(theta_object)] = theta_var_object + comp_map[theta_object] = theta_var_object - # Indexed Param - elif isinstance(param_object, IndexedParam): + # Indexed Param -- Delete Param, add Var + elif isinstance(theta_object, IndexedParam): + + # save Param values + vals = theta_object.extract_values() - print('indexed param') - # Delete Param, add Var - # Before deleting the Param, create a list of the indexed param names - vals = param_object.extract_values() - param_objects = [] - for param_obj in param_object: - indexed_param_name = param_object.name + "[" + str(param_obj) + "]" - param_cuid = ComponentUID(indexed_param_name) - param_objects.append(param_cuid.find_component_on(model)) - indexed_param_names.append(indexed_param_name) + # get indexed Params + param_theta_objects = [theta_obj for _, theta_obj in theta_object.items()] + + # get indexed Param names + indexed_param_CUIDs += [ComponentUID(theta_obj) + for _, theta_obj in theta_object.items()] - model.del_component(param_object) + # delete Param + model.del_component(theta_object) - index_name = param_object.index_set().name + # add Var w/ previous Param values + index_name = theta_object.index_set().name index_cuid = ComponentUID(index_name) index_object = index_cuid.find_component_on(model) model.add_component( - param_object.name, pyo.Var(index_object, initialize=vals) + theta_object.name, pyo.Var(index_object, initialize=vals) ) # Update substitution map (map each indexed param to indexed var) - new_var_cuid = ComponentUID(param_object.name) - new_var_object = new_var_cuid.find_component_on(model) - comp_map[param_object] = new_var_object - new_var_objects = [] - for var_obj in new_var_object: - var_cuid = ComponentUID(new_var_object.name + "[" + str(var_obj) + "]") - new_var_objects.append(var_cuid.find_component_on(model)) - - for param_obj, new_var_obj in zip(param_objects, new_var_objects): - substitution_map[id(param_obj)] = new_var_obj - comp_map[param_obj] = new_var_obj + theta_var_cuid = ComponentUID(theta_object.name) + theta_var_object = theta_var_cuid.find_component_on(model) + comp_map[theta_object] = theta_var_object + var_theta_objects = [var_theta_obj for _, var_theta_obj in theta_var_object.items()] + for param_theta_obj, var_theta_obj in zip( + param_theta_objects, var_theta_objects + ): + substitution_map[id(param_theta_obj)] = var_theta_obj + comp_map[param_theta_obj] = var_theta_obj # Var or Indexed Var - elif isinstance(param_object, IndexedVar) or param_object.is_variable_type(): - new_var_object = param_object + elif isinstance(theta_object, IndexedVar) or theta_object.is_variable_type(): + theta_var_object = theta_object else: - logger.warning("%s is not a Param or Var on the model", (str(param_object))) + logger.warning("%s is not a Param or Var on the model", (param_name)) return model if fix_vars: - new_var_object.fix() + theta_var_object.fix() else: - new_var_object.unfix() + theta_var_object.unfix() # If no substitutions are needed, return the model if len(substitution_map) == 0: return model - # Update the list of param_names if the parameters were indexed - if len(indexed_param_names) > 0: - param_names = indexed_param_names + # Update the list of param_CUIDs if the parameters were indexed + if len(indexed_param_CUIDs) > 0: + param_CUIDs = indexed_param_CUIDs # Convert Params to Vars in Expressions for expr in model.component_data_objects(pyo.Expression): From 0ae57d4f9046e3a15708d7bf95decfb35b9a377d Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Sat, 10 Aug 2024 15:29:38 -0600 Subject: [PATCH 03/11] Black formatted. --- pyomo/contrib/parmest/tests/test_utils.py | 3 +-- pyomo/contrib/parmest/utils/model_utils.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/parmest/tests/test_utils.py b/pyomo/contrib/parmest/tests/test_utils.py index bbfa90e9425..dfb3ade7aec 100644 --- a/pyomo/contrib/parmest/tests/test_utils.py +++ b/pyomo/contrib/parmest/tests/test_utils.py @@ -64,7 +64,6 @@ def test_convert_param_to_var(self): self.assertEqual(pyo.value(c), pyo.value(c_old)) self.assertTrue(c in m_vars.unknown_parameters) - # test indexed params ##################### @@ -99,7 +98,7 @@ def response_rule(m, h): model.response_function = pyo.Expression(data.hour, rule=response_rule) return model - + class RooneyBieglerExperimentIndexedParams(RooneyBieglerExperiment): def create_model(self): diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 362a7c0a218..fe4638ae987 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -81,7 +81,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # Indexed Param -- Delete Param, add Var elif isinstance(theta_object, IndexedParam): - + # save Param values vals = theta_object.extract_values() @@ -89,8 +89,9 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): param_theta_objects = [theta_obj for _, theta_obj in theta_object.items()] # get indexed Param names - indexed_param_CUIDs += [ComponentUID(theta_obj) - for _, theta_obj in theta_object.items()] + indexed_param_CUIDs += [ + ComponentUID(theta_obj) for _, theta_obj in theta_object.items() + ] # delete Param model.del_component(theta_object) @@ -107,7 +108,9 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): theta_var_cuid = ComponentUID(theta_object.name) theta_var_object = theta_var_cuid.find_component_on(model) comp_map[theta_object] = theta_var_object - var_theta_objects = [var_theta_obj for _, var_theta_obj in theta_var_object.items()] + var_theta_objects = [ + var_theta_obj for _, var_theta_obj in theta_var_object.items() + ] for param_theta_obj, var_theta_obj in zip( param_theta_objects, var_theta_objects ): @@ -150,7 +153,8 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): model.constraints = pyo.ConstraintList() for c in model.component_data_objects(pyo.Constraint): if c.active and any( - ComponentUID(v) in param_CUIDs for v in identify_mutable_parameters(c.expr) + ComponentUID(v) in param_CUIDs + for v in identify_mutable_parameters(c.expr) ): if c.equality: model.constraints.add( From 7b6449099cebbcbb6486e32ced7bc817e51ef15b Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Sat, 10 Aug 2024 15:32:00 -0600 Subject: [PATCH 04/11] Removed deprecated convert_params_to_vars function. --- pyomo/contrib/parmest/utils/model_utils.py | 184 +-------------------- 1 file changed, 1 insertion(+), 183 deletions(-) diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index fe4638ae987..edcb89f373e 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -212,186 +212,4 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # solver = pyo.SolverFactory('ipopt') # solver.solve(model) - return model - - -def convert_params_to_vars_deprecated(model, param_names=None, fix_vars=False): - """ - Convert select Params to Vars - - Parameters - ---------- - model : Pyomo concrete model - Original model - param_names : list of strings - List of parameter names to convert, if None then all Params are converted - fix_vars : bool - Fix the new variables, default is False - - Returns - ------- - model : Pyomo concrete model - Model with select Params converted to Vars - """ - - model = model.clone() - - if param_names is None: - param_names = [param.name for param in model.component_data_objects(pyo.Param)] - - indexed_param_names = [] - - # Convert Params to Vars, unfix Vars, and create a substitution map - substitution_map = {} - comp_map = ComponentMap() - for i, param_name in enumerate(param_names): - # Leverage the parser in ComponentUID to locate the component. - theta_cuid = ComponentUID(param_name) - theta_object = theta_cuid.find_component_on(model) - - # Param - if theta_object.is_parameter_type(): - # Delete Param, add Var - vals = theta_object.extract_values() - model.del_component(theta_object) - model.add_component(theta_object.name, pyo.Var(initialize=vals[None])) - - # Update substitution map - theta_var_cuid = ComponentUID(theta_object.name) - theta_var_object = theta_var_cuid.find_component_on(model) - substitution_map[id(theta_object)] = theta_var_object - comp_map[theta_object] = theta_var_object - - # Indexed Param - elif isinstance(theta_object, IndexedParam): - # Delete Param, add Var - # Before deleting the Param, create a list of the indexed param names - vals = theta_object.extract_values() - param_theta_objects = [] - for theta_obj in theta_object: - indexed_param_name = theta_object.name + '[' + str(theta_obj) + ']' - theta_cuid = ComponentUID(indexed_param_name) - param_theta_objects.append(theta_cuid.find_component_on(model)) - indexed_param_names.append(indexed_param_name) - - model.del_component(theta_object) - - index_name = theta_object.index_set().name - index_cuid = ComponentUID(index_name) - index_object = index_cuid.find_component_on(model) - model.add_component( - theta_object.name, pyo.Var(index_object, initialize=vals) - ) - - # Update substitution map (map each indexed param to indexed var) - theta_var_cuid = ComponentUID(theta_object.name) - theta_var_object = theta_var_cuid.find_component_on(model) - comp_map[theta_object] = theta_var_object - var_theta_objects = [] - for theta_obj in theta_var_object: - theta_cuid = ComponentUID( - theta_var_object.name + '[' + str(theta_obj) + ']' - ) - var_theta_objects.append(theta_cuid.find_component_on(model)) - - for param_theta_obj, var_theta_obj in zip( - param_theta_objects, var_theta_objects - ): - substitution_map[id(param_theta_obj)] = var_theta_obj - comp_map[param_theta_obj] = var_theta_obj - - # Var or Indexed Var - elif isinstance(theta_object, IndexedVar) or theta_object.is_variable_type(): - theta_var_object = theta_object - - else: - logger.warning("%s is not a Param or Var on the model", (param_name)) - return model - - if fix_vars: - theta_var_object.fix() - else: - theta_var_object.unfix() - - # If no substitutions are needed, return the model - if len(substitution_map) == 0: - return model - - # Update the list of param_names if the parameters were indexed - if len(indexed_param_names) > 0: - param_names = indexed_param_names - - # Convert Params to Vars in Expressions - for expr in model.component_data_objects(pyo.Expression): - if expr.active and any( - v.name in param_names for v in identify_mutable_parameters(expr) - ): - new_expr = replace_expressions(expr=expr, substitution_map=substitution_map) - model.del_component(expr) - model.add_component(expr.name, pyo.Expression(rule=new_expr)) - - # Convert Params to Vars in Constraint expressions - num_constraints = len(list(model.component_objects(pyo.Constraint, active=True))) - if num_constraints > 0: - model.constraints = pyo.ConstraintList() - for c in model.component_data_objects(pyo.Constraint): - if c.active and any( - v.name in param_names for v in identify_mutable_parameters(c.expr) - ): - if c.equality: - model.constraints.add( - replace_expressions( - expr=c.lower, substitution_map=substitution_map - ) - == replace_expressions( - expr=c.body, substitution_map=substitution_map - ) - ) - elif c.lower is not None: - model.constraints.add( - replace_expressions( - expr=c.lower, substitution_map=substitution_map - ) - <= replace_expressions( - expr=c.body, substitution_map=substitution_map - ) - ) - elif c.upper is not None: - model.constraints.add( - replace_expressions( - expr=c.upper, substitution_map=substitution_map - ) - >= replace_expressions( - expr=c.body, substitution_map=substitution_map - ) - ) - else: - raise ValueError( - "Unable to parse constraint to convert params to vars." - ) - c.deactivate() - - # Convert Params to Vars in Objective expressions - for obj in model.component_data_objects(pyo.Objective): - if obj.active and any( - v.name in param_names for v in identify_mutable_parameters(obj) - ): - expr = replace_expressions(expr=obj.expr, substitution_map=substitution_map) - model.del_component(obj) - model.add_component(obj.name, pyo.Objective(rule=expr, sense=obj.sense)) - - # Convert Params to Vars in Suffixes - for s in model.component_objects(pyo.Suffix): - current_keys = list(s.keys()) - for c in current_keys: - if c in comp_map: - s[comp_map[c]] = s.pop(c) - - assert len(current_keys) == len(s.keys()) - - # print('--- Updated Model ---') - # model.pprint() - # solver = pyo.SolverFactory('ipopt') - # solver.solve(model) - - return model + return model \ No newline at end of file From efa646529622a7182d1aeb6d8f72dc556eaa9eae Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Sat, 10 Aug 2024 15:32:33 -0600 Subject: [PATCH 05/11] Re-blacked. --- pyomo/contrib/parmest/utils/model_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index edcb89f373e..85542698d47 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -212,4 +212,4 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # solver = pyo.SolverFactory('ipopt') # solver.solve(model) - return model \ No newline at end of file + return model From 7ea02db686d359bf49fe33775a041a6dbebc6929 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Tue, 13 Aug 2024 12:08:06 -0600 Subject: [PATCH 06/11] Removed left over code from model_utils. --- pyomo/contrib/parmest/utils/model_utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 85542698d47..8c8066eaefb 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -48,12 +48,6 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): ComponentUID(param) for param in model.component_data_objects(pyo.Param) ] - # if param_CUIDs is None: - # param_CUIDs = [ - # ComponentUID(param.name) - # for param in model.component_data_objects(pyo.Param) - # ] - # keep a list of the parameter CUIDs in the case of indexing indexed_param_CUIDs = [] From 394a6d6d74769c36a92934228655c32269c04140 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Mon, 19 Aug 2024 10:38:10 -0600 Subject: [PATCH 07/11] Fixed efficiency bugs in parmest model_utils.py. --- pyomo/contrib/parmest/utils/model_utils.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 8c8066eaefb..74f5ea389d5 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -83,9 +83,9 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): param_theta_objects = [theta_obj for _, theta_obj in theta_object.items()] # get indexed Param names - indexed_param_CUIDs += [ - ComponentUID(theta_obj) for _, theta_obj in theta_object.items() - ] + indexed_param_CUIDs.extend( + ComponentUID(theta_obj) for theta_obj in theta_object.values() + ) # delete Param model.del_component(theta_object) @@ -102,9 +102,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): theta_var_cuid = ComponentUID(theta_object.name) theta_var_object = theta_var_cuid.find_component_on(model) comp_map[theta_object] = theta_var_object - var_theta_objects = [ - var_theta_obj for _, var_theta_obj in theta_var_object.items() - ] + var_theta_objects = list(theta_var_object.values()) for param_theta_obj, var_theta_obj in zip( param_theta_objects, var_theta_objects ): @@ -132,10 +130,14 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): if len(indexed_param_CUIDs) > 0: param_CUIDs = indexed_param_CUIDs + # convert to a set for look up efficiency + param_CUIDs_set = set(param_CUIDs) + # Convert Params to Vars in Expressions for expr in model.component_data_objects(pyo.Expression): if expr.active and any( - ComponentUID(v) in param_CUIDs for v in identify_mutable_parameters(expr) + ComponentUID(v) in param_CUIDs_set + for v in identify_mutable_parameters(expr) ): new_expr = replace_expressions(expr=expr, substitution_map=substitution_map) model.del_component(expr) @@ -147,7 +149,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): model.constraints = pyo.ConstraintList() for c in model.component_data_objects(pyo.Constraint): if c.active and any( - ComponentUID(v) in param_CUIDs + ComponentUID(v) in param_CUIDs_set for v in identify_mutable_parameters(c.expr) ): if c.equality: @@ -186,7 +188,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # Convert Params to Vars in Objective expressions for obj in model.component_data_objects(pyo.Objective): if obj.active and any( - ComponentUID(v) in param_CUIDs for v in identify_mutable_parameters(obj) + ComponentUID(v) in param_CUIDs_set for v in identify_mutable_parameters(obj) ): expr = replace_expressions(expr=obj.expr, substitution_map=substitution_map) model.del_component(obj) From 0545ecffe5860581aaf1c4069c6240b18e4a7084 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Tue, 20 Aug 2024 08:07:58 -0600 Subject: [PATCH 08/11] Bethany's changed, not including block/hierarchical model changes. --- pyomo/contrib/parmest/parmest.py | 2 +- pyomo/contrib/parmest/tests/test_utils.py | 4 ++-- pyomo/contrib/parmest/utils/model_utils.py | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/parmest/parmest.py b/pyomo/contrib/parmest/parmest.py index 1dc0b8dc66d..8d0dc89fcd1 100644 --- a/pyomo/contrib/parmest/parmest.py +++ b/pyomo/contrib/parmest/parmest.py @@ -440,7 +440,7 @@ def TotalCost_rule(model): ) # Convert theta Params to Vars, and unfix theta Vars - theta_CUIDs = [v for k, v in model.unknown_parameters.items()] + theta_CUIDs = list(model.unknown_parameters.values()) parmest_model = utils.convert_params_to_vars(model, theta_CUIDs, fix_vars=False) return parmest_model diff --git a/pyomo/contrib/parmest/tests/test_utils.py b/pyomo/contrib/parmest/tests/test_utils.py index dfb3ade7aec..ef0e3b64f91 100644 --- a/pyomo/contrib/parmest/tests/test_utils.py +++ b/pyomo/contrib/parmest/tests/test_utils.py @@ -50,7 +50,7 @@ def test_convert_param_to_var(self): exp = ReactorDesignExperiment(data, 0) instance = exp.get_labeled_model() - param_CUIDs = [v for k, v in instance.unknown_parameters.items()] + param_CUIDs = list(instance.unknown_parameters.values()) m_vars = parmest.utils.convert_params_to_vars( instance, param_CUIDs, fix_vars=True ) @@ -120,7 +120,7 @@ def label_model(self): exp = RooneyBieglerExperimentIndexedParams(self.data.loc[0, :]) instance = exp.get_labeled_model() - param_CUIDs = [v for k, v in instance.unknown_parameters.items()] + param_CUIDs = list(instance.unknown_parameters.values()) m_vars = parmest.utils.convert_params_to_vars( instance, param_CUIDs, fix_vars=True ) diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 74f5ea389d5..83d2566bbec 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -80,9 +80,9 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): vals = theta_object.extract_values() # get indexed Params - param_theta_objects = [theta_obj for _, theta_obj in theta_object.items()] - - # get indexed Param names + param_theta_objects = list(theta_object.values()) + + # get indexed Param CUIDs indexed_param_CUIDs.extend( ComponentUID(theta_obj) for theta_obj in theta_object.values() ) @@ -135,8 +135,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # Convert Params to Vars in Expressions for expr in model.component_data_objects(pyo.Expression): - if expr.active and any( - ComponentUID(v) in param_CUIDs_set + if any(ComponentUID(v) in param_CUIDs_set for v in identify_mutable_parameters(expr) ): new_expr = replace_expressions(expr=expr, substitution_map=substitution_map) From d4325007cb6e67b337764c3c1b0184808af77b77 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Tue, 20 Aug 2024 08:12:48 -0600 Subject: [PATCH 09/11] Forgot black on Bethany's changes. --- pyomo/contrib/parmest/utils/model_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index 83d2566bbec..a5e295c2e1a 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -81,7 +81,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # get indexed Params param_theta_objects = list(theta_object.values()) - + # get indexed Param CUIDs indexed_param_CUIDs.extend( ComponentUID(theta_obj) for theta_obj in theta_object.values() @@ -135,7 +135,8 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # Convert Params to Vars in Expressions for expr in model.component_data_objects(pyo.Expression): - if any(ComponentUID(v) in param_CUIDs_set + if any( + ComponentUID(v) in param_CUIDs_set for v in identify_mutable_parameters(expr) ): new_expr = replace_expressions(expr=expr, substitution_map=substitution_map) From 577aea340b3401b4341c7a8c4cd15817cf0c09c4 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Tue, 20 Aug 2024 10:39:14 -0600 Subject: [PATCH 10/11] Fixed hierarchical/block model bug. --- pyomo/contrib/parmest/tests/test_utils.py | 16 +++++++++++++++ pyomo/contrib/parmest/utils/model_utils.py | 23 ++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pyomo/contrib/parmest/tests/test_utils.py b/pyomo/contrib/parmest/tests/test_utils.py index ef0e3b64f91..e1942ddda41 100644 --- a/pyomo/contrib/parmest/tests/test_utils.py +++ b/pyomo/contrib/parmest/tests/test_utils.py @@ -135,6 +135,22 @@ def label_model(self): self.assertEqual(pyo.value(iv), pyo.value(iv_old)) self.assertTrue(c in m_vars.unknown_parameters) + # test hierarchical model + ######################### + + m = pyo.ConcreteModel() + m.p1 = pyo.Param(initialize=1, mutable=True) + m.b = pyo.Block() + m.b.p2 = pyo.Param(initialize=2, mutable=True) + + param_CUIDs = [pyo.ComponentUID(m.p1), pyo.ComponentUID(m.b.p2)] + m_vars = parmest.utils.convert_params_to_vars(m, param_CUIDs) + + for v in [str(CUID) for CUID in param_CUIDs]: + c = m_vars.find_component(v) + self.assertIsInstance(c, pyo.Var) + c_old = m.find_component(v) + self.assertEqual(pyo.value(c), pyo.value(c_old)) if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/parmest/utils/model_utils.py b/pyomo/contrib/parmest/utils/model_utils.py index a5e295c2e1a..4a405e0b318 100644 --- a/pyomo/contrib/parmest/utils/model_utils.py +++ b/pyomo/contrib/parmest/utils/model_utils.py @@ -16,6 +16,7 @@ from pyomo.core.base.var import IndexedVar from pyomo.core.base.param import IndexedParam from pyomo.common.collections import ComponentMap +from pyomo.core.base import Model from pyomo.environ import ComponentUID @@ -62,14 +63,15 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): # Param if theta_object.is_parameter_type(): - # Delete Param, add Var + # change from Param to Var vals = theta_object.extract_values() - model.del_component(theta_object) - model.add_component(theta_object.name, pyo.Var(initialize=vals[None])) + parent = theta_object.parent_block() + parent.del_component(theta_object) + parent.add_component(theta_object.name, pyo.Var(initialize=vals[None])) # Update substitution map - theta_var_cuid = ComponentUID(theta_object.name) - theta_var_object = theta_var_cuid.find_component_on(model) + theta_var_cuid = ComponentUID(theta_object.local_name) + theta_var_object = theta_var_cuid.find_component_on(parent) substitution_map[id(theta_object)] = theta_var_object comp_map[theta_object] = theta_var_object @@ -88,19 +90,20 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): ) # delete Param - model.del_component(theta_object) + parent = theta_object.parent_block() + parent.del_component(theta_object) # add Var w/ previous Param values index_name = theta_object.index_set().name index_cuid = ComponentUID(index_name) - index_object = index_cuid.find_component_on(model) - model.add_component( + index_object = index_cuid.find_component_on(parent) + parent.add_component( theta_object.name, pyo.Var(index_object, initialize=vals) ) # Update substitution map (map each indexed param to indexed var) theta_var_cuid = ComponentUID(theta_object.name) - theta_var_object = theta_var_cuid.find_component_on(model) + theta_var_object = theta_var_cuid.find_component_on(parent) comp_map[theta_object] = theta_var_object var_theta_objects = list(theta_var_object.values()) for param_theta_obj, var_theta_obj in zip( @@ -114,7 +117,7 @@ def convert_params_to_vars(model, param_CUIDs=None, fix_vars=False): theta_var_object = theta_object else: - logger.warning("%s is not a Param or Var on the model", (param_name)) + logger.warning("%s is not a Param or Var on the model", (theta_object)) return model if fix_vars: From 92c23f07a2224366b25a66e1a25aa23e2c319d19 Mon Sep 17 00:00:00 2001 From: Shawn Martin Date: Tue, 20 Aug 2024 10:41:36 -0600 Subject: [PATCH 11/11] And black. --- pyomo/contrib/parmest/tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/parmest/tests/test_utils.py b/pyomo/contrib/parmest/tests/test_utils.py index e1942ddda41..069a8748987 100644 --- a/pyomo/contrib/parmest/tests/test_utils.py +++ b/pyomo/contrib/parmest/tests/test_utils.py @@ -152,5 +152,6 @@ def label_model(self): c_old = m.find_component(v) self.assertEqual(pyo.value(c), pyo.value(c_old)) + if __name__ == "__main__": unittest.main()